Repository: hanchuanchuan/bingo2sql Branch: master Commit: 56d70d7af3cf Files: 29 Total size: 248.1 KB Directory structure: gitextract_sccp6i0i/ ├── .editorconfig ├── .gitignore ├── Makefile ├── README.md ├── circle.yml ├── cmd/ │ ├── bingo2sql/ │ │ └── main.go │ ├── local.go │ ├── remote.go │ ├── root.go │ ├── server.go │ └── stats.go ├── cnf/ │ └── config.ini ├── core/ │ ├── parseFile.go │ ├── parser.go │ ├── parserV2.go │ ├── parser_stats.go │ ├── parser_test.go │ ├── socket.go │ └── time.go ├── docs/ │ └── test.md ├── go.mod ├── go.sum ├── main.go ├── parse/ │ ├── bingo2sql.go │ └── log.go └── utils/ └── uuid/ ├── codec.go ├── generator.go ├── sql.go └── uuid.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .editorconfig ================================================ [*] end_of_line = lf insert_final_newline = true charset = utf-8 # tab_size = 4 spaces [*.go] indent_style = tab indent_size = 4 trim_trailing_whitespace = true ================================================ FILE: .gitignore ================================================ files bin coverage.out .idea/ *.iml *.swp *.log *.fail.go .DS_Store .vscode/ *.sql ================================================ FILE: Makefile ================================================ PROJECT=bingo2sql GOPATH ?= $(shell go env GOPATH) # Ensure GOPATH is set before running build process. ifeq "$(GOPATH)" "" $(error Please set the environment variable GOPATH before running `make`) endif FAIL_ON_STDOUT := awk '{ print } END { if (NR > 0) { exit 1 } }' CURDIR := $(shell pwd) path_to_add := $(addsuffix /bin,$(subst :,/bin:,$(GOPATH))) export PATH := $(path_to_add):$(PATH) GO := GO111MODULE=on go GOBUILD := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG) VERSION := $(shell git describe --tags --dirty) # 指定部分单元测试跳过 ifeq ("$(SHORT)", "1") GOTEST := CGO_ENABLED=1 $(GO) test -p 3 -short else GOTEST := CGO_ENABLED=1 $(GO) test -p 3 endif OVERALLS := CGO_ENABLED=1 GO111MODULE=on overalls GOVERALLS := goveralls ARCH := "`uname -s`" LINUX := "Linux" MAC := "Darwin" PACKAGE_LIST := go list ./...| grep -vE "vendor" PACKAGES := $$($(PACKAGE_LIST)) PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/hanchuanchuan/$(PROJECT)/||' FILES := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go" | grep -vE "vendor") GOFAIL_ENABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|vendor)" | xargs gofail enable) GOFAIL_DISABLE := $$(find $$PWD/ -type d | grep -vE "(\.git|vendor)" | xargs gofail disable) # LDFLAGS += -X "github.com/hanchuanchuan/$(PROJECT)//mysql.TiDBReleaseVersion=$(shell git describe --tags --dirty)" # LDFLAGS += -X "github.com/hanchuanchuan/$(PROJECT)//util/printer.TiDBBuildTS=$(shell date '+%Y-%m-%d %H:%M:%S')" # LDFLAGS += -X "github.com/hanchuanchuan/$(PROJECT)//util/printer.TiDBGitHash=$(shell git rev-parse HEAD)" # LDFLAGS += -X "github.com/hanchuanchuan/$(PROJECT)//util/printer.TiDBGitBranch=$(shell git rev-parse --abbrev-ref HEAD)" # LDFLAGS += -X "github.com/hanchuanchuan/$(PROJECT)//util/printer.GoVersion=$(shell go version)" CHECK_LDFLAGS += $(LDFLAGS) .PHONY: all build update clean test gotest server check default: server buildsucc build: GOOS=linux GOARCH=amd64 $(GOBUILD) -ldflags '-s -w $(LDFLAGS)' -o bin/$(PROJECT) main.go server-admin-check: server_check buildsucc buildsucc: @echo Build bingo2sql successfully! # The retool tools.json is setup from hack/retool-install.sh check-setup: @which retool >/dev/null 2>&1 || go get github.com/twitchtv/retool @retool sync check: check-setup fmt lint vet # These need to be fixed before they can be ran regularly check-fail: goword check-static check-slow fmt: @echo "gofmt (simplify)" @gofmt -s -l -w $(FILES) 2>&1 | $(FAIL_ON_STDOUT) goword: retool do goword $(FILES) 2>&1 | $(FAIL_ON_STDOUT) check-static: @ # vet and fmt have problems with vendor when ran through metalinter CGO_ENABLED=0 retool do gometalinter.v2 --disable-all --deadline 120s \ --enable misspell \ --enable megacheck \ --enable ineffassign \ $$($(PACKAGE_DIRECTORIES)) check-slow: CGO_ENABLED=0 retool do gometalinter.v2 --disable-all \ --enable errcheck \ $$($(PACKAGE_DIRECTORIES)) CGO_ENABLED=0 retool do gosec $$($(PACKAGE_DIRECTORIES)) lint: @echo "linting" @CGO_ENABLED=0 retool do revive -formatter friendly -config revive.toml $(PACKAGES) vet: @echo "vet" @go vet -all -shadow $(PACKAGES) 2>&1 | $(FAIL_ON_STDOUT) clean: $(GO) clean -i ./... rm -rf *.out test: gotest gotest: $(GO) get github.com/etcd-io/gofail@v0.0.0-20180808172546-51ce9a71510a @$(GOFAIL_ENABLE) ifeq ("$(TRAVIS_COVERAGE)", "1") @echo "Running in TRAVIS_COVERAGE mode." @export log_level=error; \ go get github.com/go-playground/overalls # go get github.com/mattn/goveralls # $(OVERALLS) -project=github.com/hanchuanchuan/$(PROJECT)/ -covermode=count -ignore='.git,vendor,cmd,docs,LICENSES' || { $(GOFAIL_DISABLE); exit 1; } # $(GOVERALLS) -service=travis-ci -coverprofile=overalls.coverprofile || { $(GOFAIL_DISABLE); exit 1; } $(OVERALLS) -project=github.com/hanchuanchuan/$(PROJECT)/ -covermode=count -ignore='.git,vendor,cmd,docs,LICENSES' -concurrency=1 -- -short || { $(GOFAIL_DISABLE); exit 1; } else @echo "Running in native mode." @export log_level=error; \ $(GOTEST) -timeout 30m -cover $(PACKAGES) || { $(GOFAIL_DISABLE); exit 1; } endif @$(GOFAIL_DISABLE) race: $(GO) get github.com/etcd-io/gofail@v0.0.0-20180808172546-51ce9a71510a @$(GOFAIL_ENABLE) @export log_level=debug; \ $(GOTEST) -timeout 30m -race $(PACKAGES) || { $(GOFAIL_DISABLE); exit 1; } @$(GOFAIL_DISABLE) leak: $(GO) get github.com/etcd-io/gofail@v0.0.0-20180808172546-51ce9a71510a @$(GOFAIL_ENABLE) @export log_level=debug; \ $(GOTEST) -tags leak $(PACKAGES) || { $(GOFAIL_DISABLE); exit 1; } @$(GOFAIL_DISABLE) tikv_integration_test: $(GO) get github.com/etcd-io/gofail@v0.0.0-20180808172546-51ce9a71510a @$(GOFAIL_ENABLE) $(GOTEST) ./store/tikv/. -with-tikv=true || { $(GOFAIL_DISABLE); exit 1; } @$(GOFAIL_DISABLE) RACE_FLAG = ifeq ("$(WITH_RACE)", "1") RACE_FLAG = -race GOBUILD = GOPATH=$(GOPATH) CGO_ENABLED=1 $(GO) build endif server: $(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS)' -o bin/$(PROJECT) main.go server_old: $(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS)' -o bin/$(PROJECT) cmd/bingo2sql/main.go server_check: $(GOBUILD) $(RACE_FLAG) -ldflags '$(CHECK_LDFLAGS)' -o bin/$(PROJECT) main.go update: which dep 2>/dev/null || go get -u github.com/golang/dep/cmd/dep ifdef PKG dep ensure -add ${PKG} else dep ensure -update endif @echo "removing test files" dep prune bash ./hack/clean_vendor.sh gofail-enable: # Converting gofail failpoints... @$(GOFAIL_ENABLE) gofail-disable: # Restoring gofail failpoints... @$(GOFAIL_DISABLE) upload-coverage: SHELL:=/bin/bash upload-coverage: ifeq ("$(TRAVIS_COVERAGE)", "1") mv overalls.coverprofile coverage.txt bash <(curl -s https://codecov.io/bash) endif # windows无法build,github.com/outbrain/golib有引用syslog.Writer,其在windows未实现. .PHONY: release release: @echo "$(CGREEN)Cross platform building for release ...$(CEND)" # @for GOOS in linux; do @for GOOS in windows darwin linux; do \ echo "Building $${GOOS}-$${GOARCH} ..."; \ GOOS=$${GOOS} GOARCH=amd64 $(GOBUILD) -ldflags '-s -w $(LDFLAGS)' -o bin/$(PROJECT) cmd/bingo2sql.go; \ cd bin; \ tar -czf $(PROJECT)-$${GOOS}-${VERSION}.tar.gz $(PROJECT); \ rm -f $(PROJECT); \ cd ..; \ done docker: GOOS=linux GOARCH=amd64 $(GOBUILD) -ldflags '-s -w $(LDFLAGS)' -o bin/$(PROJECT) cmd/bingo2sql.go v1=$(shell git tag | awk -F'-' '{print $1}' |tail -1) && docker build -t hanchuanchuan/$(PROJECT)/:$${v1} . \ && docker tag hanchuanchuan/$(PROJECT)/:$${v1} hanchuanchuan/$(PROJECT)/:latest docker-push: v1=$(shell git tag|tail -1) && docker push hanchuanchuan/$(PROJECT)/:$${v1} \ && docker push hanchuanchuan/$(PROJECT)/:latest ================================================ FILE: README.md ================================================ # bingo2sql MySQL Binlog 解析工具 从MySQL binlog解析出原始SQL，对应的回滚SQL等。 #### 功能说明 - 本地离线解析：指定本地binlog文件和要解析的表结构即可 - 远程在线解析：指定远程数据库地址，起止时间范围或binlog范围，可指定库/表和操作类型，GTID/线程号等 - 解析服务API：提供HTTP协议方式的解析接口，支持解析和打包下载 #### 限制和要求 - MySQL必须开启binlog - binlog_format = row - binlog_row_image = full #### 测试对比 测试步骤及结束详见 [效率测试](docs/test.md) ### 支持模式 #### 1. 本地解析 ```sh bingo2sql --start-file=~/db_cmdb/blog/mysql-bin.000001 -t table.sql ``` 其中`-t`参数指定的是建表语句文件,内容类似: ```sql -- 需要解析哪个表,提供哪个表的建表语句 CREATE TABLE `tt` ( id int auto_increment primary key, `TABLE_NAME` varchar(64) NOT NULL DEFAULT '' ) ; ``` #### 2. 远程解析 远程解析的参数及使用均与binlog2sql类似 ``` bingo2sql -h=127.0.0.1 -P 3306 -u test -p test -d db1_3306_test_inc \ --start-time="2006-01-02 15:04:05" -t t1 -B ``` #### 3. 解析服务 bingo2sql 支持以服务方式运行,提供解析的HTTP接口支持 ```sh bingo2sql --server --config=config.ini ``` ### 支持选项 **解析模式** - --stop-never 持续解析binlog。可选。默认false，同步至执行命令时最新的binlog位置。 - -K, --no-primary-key 对INSERT语句去除主键。可选。默认false - -B, --flashback 生成回滚SQL，可解析大文件，不受内存限制。可选。默认false。与stop-never或no-primary-key不能同时添加。 - -M, --minimal-update 最小化update语句. 可选. (default true) - -I, --minimal-insert 使用包含多个VALUES列表的多行语法编写INSERT语句. (default true) **解析范围控制** - --start-file 起始解析文件，只需文件名，无需全路径 。可选，如果指定了起止时间，可以忽略该参数。 - --start-pos 起始解析位置。可选。默认为start-file的起始位置。 - --stop-file 终止解析文件。可选。若解析模式为stop-never，此选项失效。 - --stop-pos 终止解析位置。可选。默认为stop-file的最末位置；若解析模式为stop-never，此选项失效。 - --start-time 起始解析时间，格式'%Y-%m-%d %H:%M:%S'等。可选。默认不过滤。 - --stop-time 终止解析时间，格式'%Y-%m-%d %H:%M:%S'等。可选。默认不过滤。 - -C, --connection-id 线程号，可解析指定线程的SQL。 - -g, --gtid GTID范围.格式为uuid:编号[-编号],多个时以逗号分隔，例如：6573bb29-9d94-11e9-9e0c-0242ac130002:1-100 - --max 解析的最大行数,设置为0则不限制，以避免解析范围过大 (default 100000) **对象过滤** -d, --databases 只解析目标db的sql，多个库用逗号隔开，如-d db1,db2。可选。默认为空。 -t, --tables 只解析目标table的sql，多张表用逗号隔开，如-t tbl1,tbl2。可选。默认为空。 --ddl 解析ddl，仅支持正向解析。可选。默认false。 --sql-type 只解析指定类型，支持 insert,update,delete。多个类型用逗号隔开，如--sql-type=insert,delete。可选。默认为增删改都解析。 **附加信息** - --show-gtid 显示gtid (default true) - --show-time 显示执行时间,同一时间仅显示首次 (default true) - --show-all-time 显示每条SQL的执行时间 (default false) - --show-thread 显示线程号,便于区别同一进程操作 (default false) - -o, --output 本地或远程解析时，可输出到指定文件(置空则输出到控制台，可通过 > file重定向) **mysql连接配置** (仅远程解析需要) ``` -h host -P port -u user -p password ``` #### 致谢 bingo2sql借鉴和学习了很多业界知名的开源项目，在此表示感谢！ - [go-mysql](https://github.com/siddontang/go-mysql) handle MySQL network protocol and replication - [binlog2sql](https://github.com/danfengcao/binlog2sql) Parse MySQL binlog to SQL you want - [binlog_rollback](https://github.com/GoDannyLai/binlog_rollback) mysql binlog rollback | flashback | redo | dml report | ddl info ================================================ FILE: circle.yml ================================================ version: 2 general: branches: ignore: - gh-pages jobs: build: branches: ignore: gh-pages environment: TZ: "Asia/Shanghai" docker: - image: circleci/golang:1.14 - image: circleci/mysql:5.7.31 # - image: mysql:5.7 command: mysqld --lower_case_table_names=1 --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --innodb-large-prefix=true --log-bin=on --server_id=111 --explicit_defaults_for_timestamp=true --gtid-mode=1 --enforce_gtid_consistency=1 --log-slave-updates=1 --innodb_buffer_pool_size=512M --max_allowed_packet=64M environment: MYSQL_ALLOW_EMPTY_PASSWORD: true TZ: "Asia/Shanghai" # volumes: # - /var/lib/mysql:/var/lib/mysql working_directory: /go/src/github.com/hanchuanchuan/bingo2sql steps: - run: name: Install mysql-client command: sudo apt-get update -y --allow-releaseinfo-change && sudo apt upgrade -y && sudo apt-get install default-mysql-client && sudo apt-get install --reinstall ca-certificates libgnutls30 -y - add_ssh_keys: fingerprints: - "42:94:88:f4:be:ff:0b:b9:e8:ae:05:26:f1:e7:fd:57" - checkout - run: name: Waiting for MySQL to be ready command: | for i in `seq 1 10`; do nc -z localhost 3306 && echo Success && exit 0 echo -n . sleep 1 done echo Failed waiting for MySQL && exit 1 - run: name: mysql init command: mysql -h 127.0.0.1 -u root -e "select version();create database if not exists test DEFAULT CHARACTER SET utf8;create database if not exists test_inc DEFAULT CHARACTER SET utf8;grant all on *.* to test@'127.0.0.1' identified by 'test';FLUSH PRIVILEGES;show databases;show variables like 'explicit_defaults_for_timestamp';" - run: name: "Show Variables" command: mysql -h 127.0.0.1 -u root -e "show variables" - run: rm -f go.sum - run: mysql -h 127.0.0.1 -u root -e "set global GTID_MODE = ON_PERMISSIVE;" - run: mysql -h 127.0.0.1 -u root -e "set global GTID_MODE = ON;" - run: name: "Build & Test" command: make test no_output_timeout: 1200 # - setup_remote_docker: # docker_layer_caching: false # - run: # name: Publish Docker Image to Docker Hub # command: | # if [ "${CIRCLE_BRANCH}" == "master" ]; then # echo "$DOCKERHUB_USERNAME" # echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin # make docker # make docker-push # fi ================================================ FILE: cmd/bingo2sql/main.go ================================================ package main import ( "context" "fmt" "math" "os" "sync" "github.com/hanchuanchuan/bingo2sql/core" _ "github.com/jinzhu/gorm/dialects/mysql" "github.com/pkg/profile" log "github.com/sirupsen/logrus" "github.com/spf13/pflag" ) // var parserProcess map[string]*parser.MyBinlogParser var parserProcess sync.Map type ParseInfo struct { ID string `json:"id"` ParseRows int `json:"parse_rows"` Percent int `json:"percent"` } var ( logHeader = `${time_rfc3339} ${prefix} ${level} ${short_file} ${line} ` requestHeader = `${time_rfc3339} ${remote_ip} ${method} ${uri} ${status} ${error} ${latency_human}` + "\n" flag = pflag.NewFlagSet("bingo2sql", pflag.ExitOnError) ) var ( host = flag.StringP("host", "h", "", "数据库地址") port = flag.IntP("port", "P", 3306, "端口号") user = flag.StringP("user", "u", "", "用户名") password = flag.StringP("password", "p", "", "密码") startFile = flag.String("start-file", "", "起始binlog文件") stopFile = flag.String("stop-file", "", "结束binlog文件") startPosition = flag.Int("start-pos", 0, "起始位置") stopPosition = flag.Int("stop-pos", 0, "结束位置") startTime = flag.String("start-time", "", "开始时间") stopTime = flag.String("stop-time", "", "结束时间") databases = flag.StringP("databases", "d", "", "数据库列表,多个时以逗号分隔") tables = flag.StringP("tables", "t", "", "表名或表结构文件.远程解析时指定表名(可多个,以逗号分隔),本地解析时指定表结构文件") threadID = flag.Uint64P("connection-id", "C", 0, "指定线程ID") flashback = flagBoolean("flashback", "B", false, "逆向语句") parseDDL = flagBoolean("ddl", "", false, "解析DDL语句(仅正向SQL)") sqlType = flag.String("sql-type", "insert,delete,update", "解析的语句类型") maxRows = flag.Int("max", 100000, "解析的最大行数,设置为0则不限制") threads = flag.Int("threads", 64, "解析线程数,影响文件解析速度") output = flag.StringP("output", "o", "", "输出到指定文件") gtid = flag.StringP("gtid", "g", "", "GTID范围.格式为uuid:编号[-编号],多个时以逗号分隔") // removePrimary = flagBoolean("no-primary-key", "K", false, "对INSERT语句去除主键. 可选.") minimalUpdate = flagBoolean("minimal-update", "M", true, "最小化update语句. 可选.") minimalInsert = flagBoolean("minimal-insert", "I", true, "使用包含多个VALUES列表的多行语法编写INSERT语句.") stopNever = flagBoolean("stop-never", "N", false, "持续解析binlog") showGTID = flagBoolean("show-gtid", "", true, "显示gtid") showTime = flagBoolean("show-time", "", true, "显示执行时间,同一时间仅显示首次") showAllTime = flagBoolean("show-all-time", "", false, "显示每条SQL的执行时间") showThread = flagBoolean("show-thread", "", false, "显示线程号,便于区别同一进程操作") runServer = flagBoolean("server", "s", false, "启动API服务") summary = flagBoolean("summary", "S", false, "统计binlog文件中的DML次数等信息") configFile = flag.StringP("config", "c", "config.ini", "以服务方式启动时需指定配置文件") debug = flagBoolean("debug", "", false, "调试模式,输出详细日志") mode = flag.String("profile-mode", "", "enable profiling mode, one of [cpu, mem, mutex, block]") // cpuProfile = flagBoolean("cpu-profile", "", false, "调试模式,开启CPU性能跟踪") ) func main() { flag.SortFlags = false // 隐藏CPU性能跟踪调试参数 // flag.MarkHidden("cpu-profile") if err := flag.Parse(os.Args[1:]); err != nil { log.Error(err) return } if len(os.Args) < 2 { fmt.Fprint(os.Stderr, "Usage of bingo2sql:\n") flag.PrintDefaults() return } switch *mode { case "cpu": defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop() case "mem": defer profile.Start(profile.MemProfile, profile.ProfilePath(".")).Stop() case "mutex": defer profile.Start(profile.MutexProfile, profile.ProfilePath(".")).Stop() case "block": defer profile.Start(profile.BlockProfile, profile.ProfilePath(".")).Stop() default: // do nothing } if *debug { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.ErrorLevel) } // 以独立工具运行 runParse() } // runParse 执行binlog解析 func runParse() { cfg := &core.BinlogParserConfig{ Host: *host, Port: uint16(*port), User: *user, Password: *password, StartFile: *startFile, StopFile: *stopFile, StartPosition: *startPosition, StopPosition: *stopPosition, StartTime: *startTime, StopTime: *stopTime, Flashback: *flashback, ParseDDL: *parseDDL, IncludeGtids: *gtid, Databases: *databases, Tables: *tables, SqlType: *sqlType, MaxRows: *maxRows, Threads: *threads, OutputFileStr: *output, // RemovePrimary: *removePrimary, MinimalUpdate: *minimalUpdate, MinimalInsert: *minimalInsert, ShowGTID: *showGTID, ShowTime: *showTime, ShowAllTime: *showAllTime, ShowThread: *showThread, StopNever: *stopNever, } // thread_id溢出处理 if *threadID > math.MaxUint32 { cfg.ThreadID = uint32(*threadID % (1 << 32)) } else { cfg.ThreadID = uint32(*threadID) } if p, err := core.NewBinlogParser(context.Background(), cfg); err != nil { log.Error("binlog解析操作失败") log.Error(err) return } else { err = p.Parser() if err != nil { log.Error("binlog解析操作失败") log.Error(err) return } } } func flagBoolean(name string, shorthand string, defaultVal bool, usage string) *bool { if !defaultVal { // Fix #4125, golang do not print default false value in usage, so we append it. usage = fmt.Sprintf("%s (default false)", usage) } return flag.BoolP(name, shorthand, defaultVal, usage) } ================================================ FILE: cmd/local.go ================================================ /* Copyright © 2020 NAME HERE Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "context" "math" "github.com/hanchuanchuan/bingo2sql/core" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) var threadID uint64 // rootCmd represents the base command when called without any subcommands var localCmd = &cobra.Command{ Use: "local", Short: "本地解析", Long: `指定binlog文件和表结构文件`, // Uncomment the following line if your bare application // has an action associated with it: Run: func(cmd *cobra.Command, args []string) { if cfg.Debug { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.ErrorLevel) } // thread_id溢出处理 if threadID > math.MaxUint32 { cfg.ThreadID = uint32(threadID % (1 << 32)) } else { cfg.ThreadID = uint32(threadID) } if p, err := core.NewBinlogParser(context.Background(), &cfg); err != nil { log.Error("binlog解析操作失败") log.Error(err) return } else { err = p.Parser() if err != nil { log.Error("binlog解析操作失败") log.Error(err) return } } }, } func init() { rootCmd.AddCommand(localCmd) flag := localCmd.Flags() flag.SortFlags = false initCommonFalg(flag) } ================================================ FILE: cmd/remote.go ================================================ /* Copyright © 2020 NAME HERE Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "context" "math" "github.com/hanchuanchuan/bingo2sql/core" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) // rootCmd represents the base command when called without any subcommands var remoteCmd = &cobra.Command{ Use: "remote", Short: "远程解析", Long: `远程解析需要指定数据库地址/端口等信息`, // Uncomment the following line if your bare application // has an action associated with it: Run: func(cmd *cobra.Command, args []string) { // thread_id溢出处理 if threadID > math.MaxUint32 { cfg.ThreadID = uint32(threadID % (1 << 32)) } else { cfg.ThreadID = uint32(threadID) } if p, err := core.NewBinlogParser(context.Background(), &cfg); err != nil { log.Error("binlog解析操作失败") log.Error(err) return } else { err = p.Parser() if err != nil { log.Error("binlog解析操作失败") log.Error(err) return } } }, } func init() { rootCmd.AddCommand(remoteCmd) flag := remoteCmd.Flags() flag.SortFlags = false flag.StringVarP(&cfg.Host, "host", "h", "", "host") flag.Uint16VarP(&cfg.Port, "port", "P", 3306, "port") flag.StringVarP(&cfg.User, "user", "u", "", "user") flag.StringVarP(&cfg.Password, "password", "p", "", "password") flag.Bool("help", false, "help for remote") flag.BoolVarP(&cfg.StopNever, "stop-never", "N", false, "持续解析binlog") initCommonFalg(flag) } ================================================ FILE: cmd/root.go ================================================ /* Copyright © 2020 NAME HERE Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "context" "fmt" "math" "os" "github.com/hanchuanchuan/bingo2sql/core" "github.com/pkg/profile" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" homedir "github.com/mitchellh/go-homedir" "github.com/spf13/viper" ) var mode string // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "bingo2sql", Short: "mysql binlog解析工具", Long: `用于解析mysql binlog,支持正向,逆向. 可本地或远程解析.`, // Uncomment the following line if your bare application // has an action associated with it: Run: func(cmd *cobra.Command, args []string) { // thread_id溢出处理 if threadID > math.MaxUint32 { cfg.ThreadID = uint32(threadID % (1 << 32)) } else { cfg.ThreadID = uint32(threadID) } if p, err := core.NewBinlogParser(context.Background(), &cfg); err != nil { log.Error("binlog解析操作失败") log.Error(err) return } else { err = p.Parser() if err != nil { log.Error("binlog解析操作失败") log.Error(err) return } } }, } // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() { if err := rootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) } } var cfg core.BinlogParserConfig func init() { cobra.OnInitialize(initConfig) // Here you will define your flags and configuration settings. // Cobra supports persistent flags, which, if defined here, // will be global for your application. // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.bingo2sql.yaml)") // Cobra also supports local flags, which will only run // when this action is called directly. // rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") cfg = core.BinlogParserConfig{ Port: 3306, } rootCmd.PersistentFlags().BoolVarP(&cfg.Debug, "debug", "", false, "调试模式,输出详细日志.sets log level to debug") flag := rootCmd.Flags() flag.SortFlags = false flag.StringVarP(&cfg.Host, "host", "h", "", "host") flag.Uint16VarP(&cfg.Port, "port", "P", 3306, "port") flag.StringVarP(&cfg.User, "user", "u", "", "user") flag.StringVarP(&cfg.Password, "password", "p", "", "password") flag.Bool("help", false, "help for remote") initCommonFalg(flag) flag.BoolVarP(&cfg.StopNever, "stop-never", "N", false, "持续解析binlog") } // initConfig reads in config file and ENV variables if set. func initConfig() { if cfgFile != "" { // Use config file from the flag. viper.SetConfigFile(cfgFile) } else { // Find home directory. home, err := homedir.Dir() if err != nil { fmt.Println(err) os.Exit(1) } // Search config in home directory with name ".bingo2sql" (without extension). viper.AddConfigPath(home) viper.SetConfigName(".bingo2sql") } viper.AutomaticEnv() // read in environment variables that match // If a config file is found, read it in. if err := viper.ReadInConfig(); err == nil { fmt.Println("Using config file:", viper.ConfigFileUsed()) } } func flagBoolean(flag *pflag.FlagSet, p *bool, name string, shorthand string, defaultVal bool, usage string) { if !defaultVal { // Fix #4125, golang do not print default false value in usage, so we append it. usage = fmt.Sprintf("%s (default false)", usage) } flag.BoolVarP(p, name, shorthand, defaultVal, usage) } func initCommonFalg(flag *pflag.FlagSet) { flag.StringVar(&cfg.StartFile, "start-file", "", "起始binlog文件") flag.StringVar(&cfg.StopFile, "stop-file", "", "结束binlog文件") flag.IntVar(&cfg.StartPosition, "start-pos", 0, "起始位置") flag.IntVar(&cfg.StopPosition, "stop-pos", 0, "结束位置") flag.StringVar(&cfg.StartTime, "start-time", "", "开始时间") flag.StringVar(&cfg.StopTime, "stop-time", "", "结束时间") flag.StringVarP(&cfg.Databases, "databases", "d", "", "数据库列表,多个时以逗号分隔") flag.StringVarP(&cfg.Tables, "tables", "t", "", "表名,如果数据库为多个,则需指名表前缀,多个时以逗号分隔") flag.Uint64VarP(&threadID, "connection-id", "C", 0, "指定线程ID") flag.BoolVarP(&cfg.Flashback, "flashback", "B", false, "逆向语句") flag.BoolVarP(&cfg.ParseDDL, "ddl", "", false, "解析DDL语句(仅正向SQL)") flag.StringVar(&cfg.SqlType, "sql-type", "insert,delete,update", "解析的语句类型") flag.IntVar(&cfg.MaxRows, "max", 100000, "解析的最大行数,设置为0则不限制") flag.IntVar(&cfg.Threads, "threads", 64, "解析线程数,影响文件解析速度") flag.StringVarP(&cfg.OutputFileStr, "output", "o", "", "输出到指定文件") flag.StringVarP(&cfg.IncludeGtids, "gtid", "g", "", "GTID范围.格式为uuid:编号[-编号],多个时以逗号分隔") flagBoolean(flag, &cfg.RemovePrimary, "no-primary-key", "K", false, "对INSERT语句去除主键. 可选.") flagBoolean(flag, &cfg.MinimalUpdate, "minimal-update", "M", true, "最小化update语句. 可选.") flagBoolean(flag, &cfg.MinimalInsert, "minimal-insert", "I", true, "使用包含多个VALUES列表的多行语法编写INSERT语句.") flagBoolean(flag, &cfg.ShowGTID, "show-gtid", "", true, "显示gtid") flagBoolean(flag, &cfg.ShowTime, "show-time", "", true, "显示执行时间,同一时间仅显示首次") flagBoolean(flag, &cfg.ShowAllTime, "show-all-time", "", false, "显示每条SQL的执行时间") flagBoolean(flag, &cfg.ShowThread, "show-thread", "", false, "显示线程号,便于区别同一进程操作") flag.StringVar(&mode, "profile-mode", "", "enable profiling mode, one of [cpu, mem, mutex, block]") switch mode { case "cpu": defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop() case "mem": defer profile.Start(profile.MemProfile, profile.ProfilePath(".")).Stop() case "mutex": defer profile.Start(profile.MutexProfile, profile.ProfilePath(".")).Stop() case "block": defer profile.Start(profile.BlockProfile, profile.ProfilePath(".")).Stop() } } ================================================ FILE: cmd/server.go ================================================ /* Copyright © 2020 NAME HERE Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "net/http" "os" "time" "github.com/hanchuanchuan/bingo2sql/parse" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/siddontang/go/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) var cfgFile string // rootCmd represents the base command when called without any subcommands var serverCmd = &cobra.Command{ Use: "server", Short: "以服务方式运行", Long: `用于解析mysql binlog,支持正向,逆向. 可本地或远程解析.`, // Uncomment the following line if your bare application // has an action associated with it: Run: func(cmd *cobra.Command, args []string) { startServer() }, } func init() { rootCmd.AddCommand(serverCmd) flag := serverCmd.Flags() flag.StringVarP(&cfgFile, "config", "c", "config.ini", "以服务方式启动时需指定配置文件") } func startServer() { viper := viper.New() viper.SetConfigFile(cfgFile) viper.SetConfigType("ini") if err := viper.ReadInConfig(); err != nil { log.Fatalf("Error: %s", err.Error()) return } // logDir := cnf.Section("Bingo").Key("log").String() // httpLogDir := cnf.Section("Bingo").Key("httplog").String() // level := cnf.Section("Bingo").Key("logLevel").String() logDir := viper.GetString("Bingo.log") httpLogDir := viper.GetString("Bingo.httplog") //echo's output log file elog, err := os.OpenFile(logDir, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { fmt.Println(fmt.Sprintf(`open echo log file %s error: %s`, logDir, err.Error())) return } defer elog.Close() httplog, err := os.OpenFile(httpLogDir, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) if err != nil { fmt.Println(fmt.Sprintf(`open echo log file %s error: %s`, httpLogDir, err.Error())) return } defer httplog.Close() // // 初始化Router // r := mux.NewRouter() // // // 静态文件路由 // // r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir(dir)))) // // 普通路由 // r.HandleFunc("/", HomeHandler) // r.Use(TestMiddleware) // http.ListenAndServe(":3000", r) // lvl, _ := zerolog.ParseLevel(level) // zerolog.SetGlobalLevel(lvl) // log.Logger = log.With().Caller().Logger().Output( // zerolog.ConsoleWriter{Out: elog, NoColor: true}) // log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) //new echo router := echo.New() router.Server.WriteTimeout = time.Duration(30) * time.Second // Middleware router.Use(middleware.Logger()) router.Use(middleware.Recover()) // Middleware // router.Use(middleware.LoggerWithConfig( // middleware.LoggerConfig{Output: httplog, Format: requestHeader})) // router.Use(middleware.Recover()) log.Info(`parse binlog tool is started`) group := router.Group("/binlog") group.GET("/", func(c echo.Context) error { return c.String(http.StatusOK, "Hello, World!\n") }) // 发起binlog解析请求 group.POST("/parse", parse.ParseBinlog) // 解析进程的进度 group.GET("/parse/:id", parse.GetParseInfo) // 获取所有解析 group.GET("/parse", parse.GetAllParse) // 停止解析操作 group.DELETE("/parse/:id", parse.ParseBinlogStop) // 下载解析生成的文件 group.GET("/parse/download/:id", parse.Download) // router.Logger.Fatal(router.Start(addr)) router.Logger.Fatal(router.Start(":8077")) } ================================================ FILE: cmd/stats.go ================================================ /* Copyright © 2020 NAME HERE Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "github.com/hanchuanchuan/bingo2sql/core" "github.com/pkg/profile" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) // runServer = flagBoolean(flag,, "server", "s", false, "启动API服务") // summary = flagBoolean(flag, "summary", "S", false, "统计binlog文件中的DML次数等信息") // configFile = flag.StringP("config", "c", "config.ini", "以服务方式启动时需指定配置文件") // rootCmd represents the base command when called without any subcommands var summaryCmd = &cobra.Command{ Use: "stats", Short: "stats binlog events", Long: `统计binlog文件中的DML次数等信息`, Run: func(cmd *cobra.Command, args []string) { if cfg.Debug { log.SetLevel(log.DebugLevel) } else { log.SetLevel(log.ErrorLevel) } if p, err := core.NewBinlogParserStats(&cfg); err != nil { log.Error("binlog解析操作失败") log.Error(err) return } else { err = p.ParserStats() if err != nil { log.Error("binlog解析操作失败") log.Error(err) return } } }, } func init() { rootCmd.AddCommand(summaryCmd) flag := summaryCmd.Flags() flag.SortFlags = false flag.StringVarP(&cfg.Host, "host", "h", "", "host") flag.Uint16VarP(&cfg.Port, "port", "P", 3306, "port") flag.StringVarP(&cfg.User, "user", "u", "", "user") flag.StringVarP(&cfg.Password, "password", "p", "", "password") flag.Bool("help", false, "help for remote") flag.StringVar(&cfg.StartFile, "start-file", "", "起始binlog文件") flag.StringVar(&cfg.StopFile, "stop-file", "", "结束binlog文件") flag.IntVar(&cfg.StartPosition, "start-pos", 0, "起始位置") flag.IntVar(&cfg.StopPosition, "stop-pos", 0, "结束位置") flag.StringVar(&cfg.StartTime, "start-time", "", "开始时间") flag.StringVar(&cfg.StopTime, "stop-time", "", "结束时间") flag.StringVarP(&cfg.Databases, "databases", "d", "", "数据库列表,多个时以逗号分隔") flag.StringVarP(&cfg.Tables, "tables", "t", "", "表名,如果数据库为多个,则需指名表前缀,多个时以逗号分隔") flag.BoolVarP(&cfg.ParseDDL, "ddl", "", false, "解析DDL语句(仅正向SQL)") flag.IntVar(&cfg.Threads, "threads", 64, "解析线程数,影响文件解析速度") flag.StringVarP(&cfg.OutputFileStr, "output", "o", "", "输出到指定文件") flag.StringVar(&mode, "profile-mode", "", "enable profiling mode, one of [cpu, mem, mutex, block]") switch mode { case "cpu": defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop() case "mem": defer profile.Start(profile.MemProfile, profile.ProfilePath(".")).Stop() case "mutex": defer profile.Start(profile.MutexProfile, profile.ProfilePath(".")).Stop() case "block": defer profile.Start(profile.BlockProfile, profile.ProfilePath(".")).Stop() } } ================================================ FILE: cnf/config.ini ================================================ [Bingo] addr = :8077 log = logs/go.log httplog = logs/http.log writeTimeout = 30 socketAddr = 127.0.0.1:8090 # 日志级别: debug info warn error fatal logLevel = debug # 数据库配置 [DBConfig] Host = 127.0.0.1 Port = 3306 User = test Password = test DB = dbmonitor # 数据库配置 [BackupDBConfig] Host = 127.0.0.1 Port = 20001 User = test Password = test ================================================ FILE: core/parseFile.go ================================================ package core import ( "bufio" "os" "strings" "sync" "time" "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" "github.com/mholt/archiver/v3" log "github.com/sirupsen/logrus" ) func (p *MyBinlogParser) parserFile() error { var wg sync.WaitGroup defer timeTrack(time.Now(), "parserFile") defer func() { if p.outputFile != nil { p.outputFile.Close() } }() var err error p.running = true if len(p.outputFileName) > 0 { if Exists(p.outputFileName) { // p.checkError(errors.New("指定的文件已存在!")) // 不作追加,改为文件存在时清除内容,O_APPEND p.outputFile, err = os.OpenFile(p.outputFileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) } else { p.outputFile, err = os.Create(p.outputFileName) } if err != nil { return err } p.bufferWriter = bufio.NewWriter(p.outputFile) } b := replication.NewBinlogParser() b.SetUseDecimal(true) p.currentPosition = mysql.Position{ Name: p.startFile, Pos: uint32(p.cfg.StartPosition), } sendTime := time.Now().Add(time.Second * 5) sendCount := 0 wg.Add(1) go p.ProcessChan(&wg) f := func(e *replication.BinlogEvent) error { ok, err := p.parseSingleEvent(e) if err != nil { return err } if !ok { b.Stop() } if e.Header.EventType == replication.QUERY_EVENT || e.Header.EventType == replication.UPDATE_ROWS_EVENTv2 || e.Header.EventType == replication.WRITE_ROWS_EVENTv2 || e.Header.EventType == replication.DELETE_ROWS_EVENTv2 { if len(p.cfg.SocketUser) > 0 { // 如果指定了websocket用户,就每5s发送一次进度通知 if p.changeRows > sendCount && time.Now().After(sendTime) { sendCount = p.changeRows sendTime = time.Now().Add(time.Second * 5) kwargs := map[string]interface{}{"rows": p.changeRows} if p.stopTimestamp > 0 && p.startTimestamp > 0 && p.stopTimestamp > p.startTimestamp { kwargs["pct"] = (e.Header.Timestamp - p.startTimestamp) * 100 / (p.stopTimestamp - p.startTimestamp) } go sendMsg(p.cfg.SocketUser, "binlog_parse_progress", "binlog解析进度", "", kwargs) } } if p.cfg.MaxRows > 0 && p.changeRows >= p.cfg.MaxRows { log.Info("已超出最大行数") if !p.cfg.StopNever { b.Stop() return nil } } } // 再次判断是否到结束位置,以免处在临界值,且无新日志时程序卡住 if e.Header.Timestamp > 0 { if p.stopTimestamp > 0 && e.Header.Timestamp >= p.stopTimestamp { log.Warn("已超出结束时间") b.Stop() return nil } } return nil } err = b.ParseFile(p.startFile, int64(p.cfg.StartPosition), f) if err != nil { println(err.Error()) } log.Info("操作完成") close(p.ch) wg.Wait() if len(p.cfg.SocketUser) > 0 { if p.changeRows > 0 { kwargs := map[string]interface{}{"rows": p.changeRows} kwargs["pct"] = 99 go sendMsg(p.cfg.SocketUser, "binlog_parse_progress", "binlog解析进度", "", kwargs) var url string if strings.Count(p.outputFileName, ".") > 0 { a := strings.Split(p.outputFileName, ".") url = strings.Join(a[0:len(a)-1], ".") } else { url = p.outputFileName } url = url + ".tar.gz" // err := archiver.TarGz.Make(url, []string{p.outputFileName}) // err := archiver.TarGz.Archiver( // []string{p.outputFileName}, // url) err := archiver.Archive([]string{p.outputFileName}, url) if err != nil { return err } fileInfo, _ := os.Stat(url) //文件大小 filesize := fileInfo.Size() // 压缩完成后删除文件 p.clear() log.Info("压缩完成") kwargs = map[string]interface{}{"ok": "1", "pct": 100, "rows": p.changeRows, "url": "/go/download/" + strings.Replace(url, "../", "", -1), "size": filesize} go sendMsg(p.cfg.SocketUser, "binlog_parse_progress", "binlog解析进度", "", kwargs) } else { // 没有数据时清除文件 p.clear() kwargs := map[string]interface{}{"ok": "1", "size": 0, "pct": 100, "rows": 0} go sendMsg(p.cfg.SocketUser, "binlog_parse_progress", "binlog解析进度", "", kwargs) } } log.Info("操作结束") return nil } ================================================ FILE: core/parser.go ================================================ package core import ( "bufio" "bytes" "context" "database/sql/driver" "errors" "fmt" "io" "math" "os" "strconv" "strings" "sync" "time" "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" uuid "github.com/hanchuanchuan/bingo2sql/utils/uuid" "github.com/jinzhu/gorm" "github.com/jinzhu/now" "github.com/mholt/archiver/v3" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" "github.com/hanchuanchuan/goInception/ast" tidb "github.com/hanchuanchuan/goInception/mysql" tidbParser "github.com/hanchuanchuan/goInception/parser" ) const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" // Column 列结构 type Column struct { gorm.Model ColumnName string `gorm:"Column:COLUMN_NAME"` CollationName string `gorm:"Column:COLLATION_NAME"` CharacterSetName string `gorm:"Column:CHARACTER_SET_NAME"` ColumnComment string `gorm:"Column:COLUMN_COMMENT"` ColumnType string `gorm:"Column:COLUMN_TYPE"` ColumnKey string `gorm:"Column:COLUMN_KEY"` Extra string `gorm:"Column:EXTRA"` isGenerated *bool `gorm:"-"` } // IsGenerated 是否为计算列 func (f *Column) IsGenerated() bool { if f.isGenerated == nil { v := strings.Contains(f.Extra, "VIRTUAL GENERATED") || strings.Contains(f.Extra, "STORED GENERATED") f.isGenerated = &v } return *f.isGenerated } // IsUnsigned 是否无符号列 func (f *Column) IsUnsigned() bool { columnType := f.ColumnType if strings.Contains(columnType, "unsigned") || strings.Contains(columnType, "zerofill") { return true } return false } // Table 表结构 type Table struct { tableID uint64 // 仅本地解析使用 TableName string Schema string Columns []Column hasPrimary bool primarys map[int]bool } // ValidColumns 可用列 func (t *Table) ValidColumns() (result []Column) { if t == nil { return } for _, f := range t.Columns { if !f.IsGenerated() { result = append(result, f) } } return result } // MasterStatus 主库binlog信息（相对bingo2sql的主库，可以是只读库） type MasterStatus struct { gorm.Model File string `gorm:"Column:File"` Position int `gorm:"Column:Position"` Binlog_Do_DB string `gorm:"Column:Binlog_Do_DB"` Binlog_Ignore_DB string `gorm:"Column:Binlog_Ignore_DB"` Executed_Gtid_Set string `gorm:"Column:Executed_Gtid_Set"` } // MasterLog 主库binlog日志 type MasterLog struct { gorm.Model Name string `gorm:"Column:Log_name"` Size int `gorm:"Column:File_size"` } // GtidSetInfo GTID集合,可指定或排除GTID范围 type GtidSetInfo struct { uuid []byte startSeqNo int64 stopSeqNo int64 } // BinlogParserConfig 解析选项 type BinlogParserConfig struct { Flavor string // InsID int `json:"ins_id" form:"ins_id"` Host string `json:"host" form:"host"` Port uint16 `json:"port" form:"port"` User string `json:"user" form:"user"` Password string `json:"password" form:"password"` // binlog文件,位置 StartFile string `json:"start_file" form:"start_file"` StopFile string `json:"stop_file" form:"stop_file"` StartPosition int `json:"start_position" form:"start_position"` StopPosition int `json:"stop_position" form:"stop_position"` // 起止时间 StartTime string `json:"start_time" form:"start_time"` StopTime string `json:"stop_time" form:"stop_time"` // 回滚 Flashback bool `json:"flashback" form:"flashback"` // 解析DDL语句 ParseDDL bool `json:"parse_ddl" form:"parse_ddl"` // 限制的数据库 Databases string `json:"databases" form:"databases"` // 限制的表 Tables string `json:"tables" form:"tables"` // sql类型 SqlType string `json:"sql_type" form:"sql_type"` // 最大解析行数 MaxRows int `json:"max_rows" form:"max_rows"` // 解析线程数,影响文件解析速度 Threads int `json:"threads" form:"threads"` // 输出到指定文件.为空时输出到控制台 OutputFileStr string // 输出所有列(已弃用,有主键时均会使用主键) // AllColumns bool `json:"all_columns" form:"all_columns"` // debug模式,打印详细日志 Debug bool `json:"debug" form:"debug"` // websocket的用户名,用以发送进度提醒 SocketUser string `json:"socket_user" form:"socket_user"` // 解析任务开始时间 beginTime int64 ThreadID uint32 `json:"thread_id" form:"thread_id"` IncludeGtids string `json:"include_gtids" form:"include_gtids"` // 对INSERT语句去除主键,以便于使用. 默认为false RemovePrimary bool // 持续解析binlog StopNever bool // 最小化Update语句, 当开启时,update语句中未变更的值不再记录到解析结果中 MinimalUpdate bool // 使用包含多个VALUES列表的多行语法编写INSERT语句 MinimalInsert bool // 输出设置 ShowGTID bool // 输出GTID ShowTime bool // 输出执行时间(相同时间时仅返回首次) ShowAllTime bool // 输出所有执行时间 ShowThread bool // 输出线程号,方便判断同一执行人的操作 uniqueKey string } type Parser interface { write([]byte, *replication.BinlogEvent) } // row 协程解析binlog事件的通道 type row struct { sql []byte e *replication.BinlogEvent gtid []byte opid string // 用以打印线程号,不打印时置空 threadID uint32 } type baseParser struct { ctx context.Context cancelFn context.CancelFunc // 记录表结构以重用 allTables map[uint64]*Table cfg *BinlogParserConfig startFile string stopFile string // binlog数据库 db *gorm.DB running bool lastTimestamp uint32 write1 Parser ch chan *row // currentBackupInfo BackupInfo // gtid []byte // write interface{} OnlyDatabases map[string]bool OnlyTables map[string]string SqlTypes map[string]bool // 本地解析模式.指定binlog文件和表结构本地解析 localMode bool } // MyBinlogParser 解析类 type MyBinlogParser struct { baseParser masterStatus *MasterStatus startTimestamp uint32 stopTimestamp uint32 // 输出到指定文件 outputFile *os.File outputFileName string // db *gorm.DB // 使用buffer写的测试 // 解析600M日志,约500w行,直接使用文件时,用时5min // 使用buffer用时1min bufferWriter *bufio.Writer // lastTimestamp uint32 gtid []byte lastGtid []byte gtidEvent *replication.GTIDEvent includeGtids []*GtidSetInfo jumpGtids map[*GtidSetInfo]bool // 表结构缓存(仅用于本地解析) tableCacheList map[string]*Table // 解析用临时变量 currentPosition mysql.Position // 当前binlog位置 currentThreadID uint32 // 当前event线程号 changeRows int // 解析SQL行数 currentTimtstamp uint32 // 当前解析到的时间戳 // 保存binlogs文件和position,用以计算percent binlogs []MasterLog // 表结构发生变化后导致无法正确解析的列表 ignoreTables map[string]interface{} } var ( TimeFormat string = "2006-01-02 15:04:05" TimeLocation *time.Location ) func init() { TimeLocation, _ = time.LoadLocation("Asia/Shanghai") } // SetUniqueKey 自定义唯一标识, 指定后通过ID()将返回该值 func (cfg *BinlogParserConfig) SetUniqueKey(key string) { cfg.uniqueKey = key } // ID 解析任务的唯一标识 func (cfg *BinlogParserConfig) ID() string { if cfg.uniqueKey != "" { return cfg.uniqueKey } s1 := strings.Replace(cfg.Host, ".", "_", -1) if len(s1) > 20 { s1 = s1[:20] } var s2 string // if len(cfg.StartTime) > 0 { // s2 = strings.Replace(cfg.StartTime, ".", "_", -1) // s2 = strings.Replace(s2, " ", "_", -1) // s2 = strings.Replace(s2, ":", "", -1) // } else { // s2 = strconv.FormatInt(cfg.beginTime, 10) // } s2 = strconv.FormatInt(cfg.beginTime, 10) if cfg.Flashback { s2 += "_rollback" } return fmt.Sprintf("%s_%d_%s", s1, cfg.Port, s2) } // Parser 远程解析 func (p *MyBinlogParser) Parser() error { // runtime.GOMAXPROCS(runtime.NumCPU()) if p.cfg.Host == "" && p.cfg.StartFile != "" { _, err := os.Stat(p.cfg.StartFile) if err != nil { return fmt.Errorf("open start_file: %w", err) } return p.parserFile() } var wg sync.WaitGroup defer timeTrack(time.Now(), "Parser") defer func() { if p.outputFile != nil { p.outputFile.Close() } }() var err error p.running = true p.db, err = p.getDB() if err != nil { return err } defer p.db.Close() if len(p.outputFileName) > 0 { if Exists(p.outputFileName) { // p.checkError(errors.New("指定的文件已存在!")) // 不作追加,改为文件存在时清除内容,O_APPEND p.outputFile, err = os.OpenFile(p.outputFileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) } else { p.outputFile, err = os.Create(p.outputFileName) } if err != nil { return fmt.Errorf("create output file: %w", err) } p.bufferWriter = bufio.NewWriter(p.outputFile) } cfg := replication.BinlogSyncerConfig{ ServerID: 2000000111, Flavor: p.cfg.Flavor, Host: p.cfg.Host, Port: p.cfg.Port, User: p.cfg.User, Password: p.cfg.Password, UseDecimal: true, // RawModeEnabled: p.cfg.RawMode, // SemiSyncEnabled: p.cfg.SemiSync, } b := replication.NewBinlogSyncer(cfg) defer b.Close() p.currentPosition = mysql.Position{ Name: p.startFile, Pos: uint32(p.cfg.StartPosition), } s, err := b.StartSync(p.currentPosition) if err != nil { log.Infof("Start sync error: %v\n", err) return fmt.Errorf("Start sync: %w", err) } sendTime := time.Now().Add(time.Second * 5) sendCount := 0 wg.Add(1) go p.ProcessChan(&wg) // var ctx context.Context var finishError error FOR: for { // if p.cfg.StopNever { // ctx = p.ctx // } else { // var cancel context.CancelFunc // ctx, cancel = context.WithTimeout(p.ctx, 10*time.Second) // defer cancel() // } // e, err := s.GetEvent(context.Background()) e, err := s.GetEvent(p.ctx) if err != nil { // Try to output all left events // events := s.DumpEvents() // for _, e := range events { // // e.Dump(os.Stdout) // log.Info("===============") // log.Info(e.Header.EventType) // } if err == context.DeadlineExceeded { log.Warnf("Waiting for timeout(10s), no new event is generated, automatically stop: %v\n", err) // err = fmt.Errorf("Waiting for timeout(10s), no new event is generated, automatically stop: %v\n", err) err = nil } else { log.Errorf("Get event error: %v\n", err) } finishError = fmt.Errorf("get binlog event: %w", err) break FOR } select { case <-p.ctx.Done(): finishError = context.Canceled break FOR default: ok, err := p.parseSingleEvent(e) if err != nil { finishError = err break FOR } if !ok { break FOR } // if time.Now().After(sendTime) { // var m runtime.MemStats // runtime.ReadMemStats(&m) // log.Debugf("%d MB\n", m.Alloc/1024/1024) // sendTime = time.Now().Add(time.Second * 1) // } if len(p.cfg.SocketUser) > 0 { // 如果指定了websocket用户,就每5s发送一次进度通知 if p.changeRows > sendCount && time.Now().After(sendTime) { sendCount = p.changeRows sendTime = time.Now().Add(time.Second * 5) kwargs := map[string]interface{}{"rows": p.changeRows} if p.stopTimestamp > 0 && p.startTimestamp > 0 && p.stopTimestamp > p.startTimestamp { kwargs["pct"] = (e.Header.Timestamp - p.startTimestamp) * 100 / (p.stopTimestamp - p.startTimestamp) } go sendMsg(p.cfg.SocketUser, "binlog_parse_progress", "binlog解析进度", "", kwargs) } } } } close(p.ch) wg.Wait() if len(p.cfg.SocketUser) > 0 { if p.changeRows > 0 { kwargs := map[string]interface{}{"rows": p.changeRows} kwargs["pct"] = 99 go sendMsg(p.cfg.SocketUser, "binlog_parse_progress", "binlog解析进度", "", kwargs) // 打包,以便下载 fileSize, err := p.Archive() if err != nil { return err } kwargs = map[string]interface{}{ "ok": "1", "pct": 100, "rows": p.changeRows, "size": fileSize} go sendMsg(p.cfg.SocketUser, "binlog_parse_progress", "binlog解析进度", "", kwargs) } else { // 没有数据时清除文件 p.clear() kwargs := map[string]interface{}{"ok": "1", "size": 0, "pct": 100, "rows": 0} go sendMsg(p.cfg.SocketUser, "binlog_parse_progress", "binlog解析进度", "", kwargs) } } p.running = false log.WithField("parsed_rows", p.changeRows).Info("解析完成") return finishError } // checkFinish 检查是否需要结束循环 func (p *MyBinlogParser) checkFinish(currentPosition *mysql.Position) int { returnValue := -1 // 满足以下任一条件时结束循环 // * binlog文件超出指定结束文件 // * 超出指定结束文件的指定位置 // * 超出当前最新binlog位置 // 当前binlog解析位置 相对于配置的结束位置/最新位置 // 超出时返回 1 // 等于时返回 0 // 小于时返回 -1 // 根据是否为事件开始做不同判断 // 事件开始时做大于判断 // 事件结束时做等于判断 var stopMsg string if p.stopFile != "" && currentPosition.Name > p.stopFile { stopMsg = "超出指定结束文件" returnValue = 1 } else if p.cfg.StopPosition > 0 && currentPosition.Name == p.stopFile { if currentPosition.Pos > uint32(p.cfg.StopPosition) { stopMsg = "超出指定位置" returnValue = 1 } else if currentPosition.Pos == uint32(p.cfg.StopPosition) { stopMsg = "超出指定位置" returnValue = 0 } } if p.masterStatus != nil { if currentPosition.Name > p.masterStatus.File || currentPosition.Name == p.masterStatus.File && currentPosition.Pos > uint32(p.masterStatus.Position) { stopMsg = "超出最新binlog位置" returnValue = 1 } else if currentPosition.Name == p.masterStatus.File && currentPosition.Pos == uint32(p.masterStatus.Position) { stopMsg = "超出最新binlog位置" returnValue = 0 } } if stopMsg != "" { log.WithFields(log.Fields{ "当前文件": currentPosition.Name, "结束文件": p.stopFile, "当前位置": currentPosition.Pos, "结束位置": p.cfg.StopPosition, }).Info(stopMsg) } return returnValue } // clear 压缩完成或者没有数据时删除文件 func (p *MyBinlogParser) clear() { err := os.Remove(p.outputFileName) if err != nil { log.Error(err) log.Errorf("删除文件失败! %s", p.outputFileName) } } // isGtidEventInGtidSet 检查gtid事件 // 0 正常,包含 // 1 跳转,不包含 // 2 超出结束位置,跳过 func (p *MyBinlogParser) isGtidEventInGtidSet() (status uint8) { e := p.gtidEvent // 0 正常,包含 // 1 跳转,不包含 // 2 超出结束位置,跳过 if len(p.includeGtids) == 0 { status = 0 return } if e == nil { status = 0 return } for _, info := range p.includeGtids { if ok, _ := p.jumpGtids[info]; !ok && bytes.Equal(e.SID, info.uuid) { if e.GNO < info.startSeqNo { status = 1 return } else if e.GNO > info.stopSeqNo { p.jumpGtids[info] = true } else { status = 0 return } } } all_ok := true for _, value := range p.jumpGtids { if !value { all_ok = false } } if all_ok { status = 2 return } status = 1 return } func (p *MyBinlogParser) Stop() { log.Warn("Process killed") if p.cancelFn != nil { p.cancelFn() } p.running = false } func (p *MyBinlogParser) write(b []byte, binEvent *replication.BinlogEvent) { data := &row{ sql: b, e: binEvent, } if p.Config().ShowThread { data.threadID = p.currentThreadID } if p.Config().ShowGTID { data.gtid = p.gtid } p.ch <- data } // byteEquals 判断字节切片是否相等 func byteEquals(v1, v2 []byte) bool { if len(v1) != len(v2) { return false } for i, v := range v1 { if v != v2[i] { return false } } return true } func (p *MyBinlogParser) myWrite(data *row) { var buf bytes.Buffer // 输出GTID if p.Config().ShowGTID && len(data.gtid) > 0 { if len(p.lastGtid) == 0 { p.lastGtid = data.gtid buf.WriteString("# ") buf.Write(data.gtid) buf.WriteString("\n") } else if !byteEquals(data.gtid, p.lastGtid) { p.lastGtid = data.gtid buf.WriteString("\n# ") buf.Write(data.gtid) buf.WriteString("\n") } } buf.Write(data.sql) if p.Config().ShowAllTime { timeinfo := fmt.Sprintf("; # %s", time.Unix(int64(data.e.Header.Timestamp), 0).Format(TimeFormat)) buf.WriteString(timeinfo) if p.Config().ShowThread { buf.WriteString(" # thread_id=") buf.WriteString(strconv.Itoa(int(data.threadID))) } buf.WriteString("\n") } else if p.Config().ShowTime { if p.lastTimestamp != data.e.Header.Timestamp { timeinfo := fmt.Sprintf("; # %s", time.Unix(int64(data.e.Header.Timestamp), 0).Format(TimeFormat)) buf.WriteString(timeinfo) p.lastTimestamp = data.e.Header.Timestamp } else { buf.WriteString(";") } if p.Config().ShowThread { buf.WriteString(" # thread_id=") buf.WriteString(strconv.Itoa(int(data.threadID))) } buf.WriteString("\n") } else { if p.Config().ShowThread { buf.WriteString(" # thread_id=") buf.WriteString(strconv.Itoa(int(data.threadID))) } else { buf.WriteString(";") } buf.WriteString("\n") } p.write2(buf.Bytes()) } func (p *MyBinlogParser) write2(b []byte) { if len(p.outputFileName) > 0 { p.bufferWriter.Write(b) } else { fmt.Print(string(b)) } } // NewBinlogParser binlog解析器 func NewBinlogParser(ctx context.Context, cfg *BinlogParserConfig) (*MyBinlogParser, error) { p := new(MyBinlogParser) p.allTables = make(map[uint64]*Table) p.jumpGtids = make(map[*GtidSetInfo]bool) p.ch = make(chan *row, cfg.Threads) p.write1 = p cfg.beginTime = time.Now().Unix() p.cfg = cfg p.OnlyDatabases = make(map[string]bool) // [table_name] = db_name p.OnlyTables = make(map[string]string) p.SqlTypes = make(map[string]bool) p.ignoreTables = make(map[string]interface{}) p.startFile = cfg.StartFile p.stopFile = cfg.StopFile if err := p.parseGtidSets(); err != nil { return nil, err } if len(cfg.SocketUser) > 0 { p.outputFileName = fmt.Sprintf("files/%s.sql", p.cfg.ID()) // p.outputFileName = fmt.Sprintf("files/%s_%s.sql", time.Now().Format("20060102_1504"), p.cfg.Id()) // var fileName []string // fileName = append(fileName, strings.Replace(cfg.Host, ".", "_", -1)) // fileName = append(fileName, strconv.Itoa(int(cfg.Port))) // if cfg.Databases != "" { // fileName = append(fileName, cfg.Databases) // } // if cfg.Tables != "" { // fileName = append(fileName, cfg.Tables) // } // fileName = append(fileName, time.Now().Format("20060102_1504")) // if cfg.Flashback { // fileName = append(fileName, "rollback") // } // p.outputFileName = fmt.Sprintf("files/%s.sql", strings.Join(fileName, "_")) } else { if cfg.OutputFileStr != "" { p.outputFileName = cfg.OutputFileStr } else { p.outputFile = os.Stdout } } // 本地解析模式,host为空,表名为SQL文件 if p.cfg.Host == "" { if p.cfg.Tables == "" { return nil, fmt.Errorf("本地解析模式请指定表结构文件--tables") } _, err := os.Stat(p.cfg.Tables) if err != nil { return nil, fmt.Errorf("读取表结构文件失败(%s): %v", p.cfg.Tables, err) } if err := p.readTableSchema(p.cfg.Tables); err != nil { return nil, fmt.Errorf("读取表结构文件失败(%s): %v", p.cfg.Tables, err) } if len(p.tableCacheList) == 0 { return nil, fmt.Errorf("未找到建表语句! 请提供需要解析的表对应的建表语句,并以分号分隔.") } else { log.Infof("共读取到表结构 %d 个", len(p.tableCacheList)) } p.localMode = true } if !p.localMode { var err error p.db, err = p.getDB() if err != nil { return nil, err } defer p.db.Close() p.masterStatus, err = p.mysqlMasterStatus() if err != nil { return nil, err } if p.cfg.Debug { p.db.LogMode(true) } } if err := p.parserInit(); err != nil { return nil, err } p.ctx, p.cancelFn = context.WithCancel(ctx) return p, nil } func (p *MyBinlogParser) ProcessChan(wg *sync.WaitGroup) { for { r := <-p.ch if r == nil { if len(p.outputFileName) > 0 { p.bufferWriter.Flush() } wg.Done() break } // p.myWrite(r.sql, r.e, r.gtid) p.myWrite(r) } } // parseGtidSets 解析gtid集合 func (p *MyBinlogParser) parseGtidSets() error { if len(p.cfg.IncludeGtids) == 0 { return nil } sets := strings.Split(p.cfg.IncludeGtids, ",") for _, s := range sets { g := &GtidSetInfo{} str := strings.Split(s, ":") if len(str) != 2 { return errors.New("错误GTID格式!正确格式为uuid:编号[-编号],多个时以逗号分隔") } // 把uuid逆向为16位[]byte u2, err := uuid.FromString(str[0]) if err != nil { return fmt.Errorf("GTID解析失败!(%s)", err.Error()) } g.uuid = u2.Bytes() nos := strings.Split(str[1], "-") if len(nos) == 1 { n, err := strconv.ParseInt(nos[0], 10, 64) if err != nil { return fmt.Errorf("GTID解析失败!(%s)", err.Error()) } g.startSeqNo = n g.stopSeqNo = n } else { n, err := strconv.ParseInt(nos[0], 10, 64) if err != nil { return fmt.Errorf("GTID解析失败!(%s)", err.Error()) } n2, err := strconv.ParseInt(nos[1], 10, 64) if err != nil { return fmt.Errorf("GTID解析失败!(%s)", err.Error()) } g.startSeqNo = n g.stopSeqNo = n2 } p.jumpGtids[g] = false p.includeGtids = append(p.includeGtids, g) // fmt.Println(*g) } log.Debugf("gtid集合数量: %d", len(p.includeGtids)) return nil } func (p *baseParser) getDB() (*gorm.DB, error) { if p.cfg.Host == "" { return nil, fmt.Errorf("请指定数据库地址") } if p.cfg.Port == 0 { return nil, fmt.Errorf("请指定数据库端口") } if p.cfg.User == "" { return nil, fmt.Errorf("请指定数据库用户名") } addr := fmt.Sprintf("%s:%s@tcp(%s:%d)/mysql?charset=utf8mb4&parseTime=True&loc=Local", p.cfg.User, p.cfg.Password, p.cfg.Host, p.cfg.Port) return gorm.Open("mysql", addr) } // parserInit 解析服务初始化 func (p *MyBinlogParser) parserInit() error { defer timeTrack(time.Now(), "parserInit") if len(p.outputFileName) > 0 { var err error if Exists(p.outputFileName) { // p.checkError(errors.New("指定的文件已存在!")) // 不作追加,改为文件存在时清除内容,O_APPEND p.outputFile, err = os.OpenFile(p.outputFileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) } else { p.outputFile, err = os.Create(p.outputFileName) } p.checkError(err) // outputFile, err := os.OpenFile(outputFileStr, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) // if err != nil { // log.Info(index, "open file failed.", err.Error()) // break // } defer p.outputFile.Close() p.bufferWriter = bufio.NewWriter(p.outputFile) } if p.cfg.StartTime != "" { t, err := now.Parse(p.cfg.StartTime) if err != nil { return err } p.startTimestamp = uint32(t.Unix()) } if p.cfg.StopTime != "" { t, err := now.Parse(p.cfg.StopTime) if err != nil { return err } p.stopTimestamp = uint32(t.Unix()) } // 如果未指定开始文件,就自动解析 if len(p.startFile) == 0 { binlogs := p.autoParseBinlogPosition() if len(binlogs) == 0 { p.checkError(errors.New("无法获取master binlog")) } p.startFile = binlogs[0].Name if p.startTimestamp > 0 || p.stopTimestamp > 0 { for _, masterLog := range binlogs { timestamp, err := p.getBinlogFirstTimestamp(masterLog.Name) p.checkError(err) log.WithFields(log.Fields{ "起始时间": time.Unix(int64(timestamp), 0).Format(TimeFormat), "binlogFile": masterLog.Name, }).Info("binlog信息") if timestamp <= p.startTimestamp { p.startFile = masterLog.Name } if p.stopFile == "" && p.stopTimestamp > 0 && timestamp > p.stopTimestamp { p.stopFile = masterLog.Name } } } if len(p.startFile) == 0 { p.checkError(errors.New("未能解析指定时间段的binlog文件,请检查后重试!")) } log.Infof("根据指定的时间段,解析出的开始binlog文件是:%s,结束文件是:%s\n", p.startFile, p.stopFile) } // // 未指定结束文件时,仅解析该文件 // if len(p.stopFile) == 0 { // p.stopFile = p.startFile // } if len(p.cfg.SqlType) > 0 { for _, s := range strings.Split(p.cfg.SqlType, ",") { p.SqlTypes[s] = true } } else { p.SqlTypes["insert"] = true p.SqlTypes["update"] = true p.SqlTypes["delete"] = true } if len(p.cfg.Databases) > 0 { for _, db := range strings.Split(p.cfg.Databases, ",") { db = strings.ToLower(strings.Trim(db, " `")) p.OnlyDatabases[db] = true } } if len(p.cfg.Tables) > 0 { for _, s := range strings.Split(p.cfg.Tables, ",") { if strings.Contains(s, ".") { names := strings.SplitN(s, ".", 2) db, table := names[0], names[1] db = strings.ToLower(strings.Trim(db, " `")) table = strings.ToLower(strings.Trim(table, " `")) p.OnlyTables[table] = db } else { key := strings.ToLower(strings.Trim(s, " `")) p.OnlyTables[key] = "" } } } // log.Infof("dbs: %#v", p.OnlyDatabases) // log.Infof("tbls: %#v", p.OnlyTables) return nil } func Exists(filename string) bool { _, err := os.Stat(filename) return err == nil || os.IsExist(err) } // getBinlogFirstTimestamp 获取binlog文件第一个时间戳 func (p *MyBinlogParser) getBinlogFirstTimestamp(file string) (uint32, error) { logLevel := log.GetLevel() defer func() { log.SetLevel(logLevel) }() // 临时调整日志级别,忽略close sync异常 log.SetLevel(log.FatalLevel) cfg := replication.BinlogSyncerConfig{ ServerID: 2000000110, Flavor: p.cfg.Flavor, Host: p.cfg.Host, Port: p.cfg.Port, User: p.cfg.User, Password: p.cfg.Password, // RawModeEnabled: p.cfg.RawMode, // SemiSyncEnabled: p.cfg.SemiSync, } b := replication.NewBinlogSyncer(cfg) pos := mysql.Position{Name: file, Pos: uint32(4)} s, err := b.StartSync(pos) if err != nil { return 0, err } defer func() { b.Close() }() for { // return e, err := s.GetEvent(context.Background()) if err != nil { // b.Close() return 0, err } // log.Infof("事件类型:%s", e.Header.EventType) if e.Header.Timestamp > 0 { // b.Close() return e.Header.Timestamp, nil } } } // check 读取文件的函数调用大多数都需要检查错误， // 使用下面这个错误检查方法可以方便一点 func check(e error) { if e != nil { panic(e) } } func timeParseToUnix(timeStr string) (uint32, error) { // t, err := time.Parse(TimeFormat, *timeStr) t, err := time.ParseInLocation(TimeFormat, timeStr, TimeLocation) if err != nil { return 0, err } stamp := t.Unix() return uint32(stamp), nil } func (p *MyBinlogParser) checkError(e error) { if e != nil { log.Error(e) if len(p.cfg.SocketUser) > 0 { kwargs := map[string]interface{}{"error": e.Error()} sendMsg(p.cfg.SocketUser, "binlog_parse_progress", "binlog解析进度", "", kwargs) } panic(e) } } func (p *MyBinlogParser) schemaFilter(table *replication.TableMapEvent) bool { if len(p.OnlyDatabases) == 0 && len(p.OnlyTables) == 0 { return true } dbName := strings.ToLower(string(table.Schema)) if len(p.OnlyDatabases) > 0 { if _, ok := p.OnlyDatabases[dbName]; !ok { return false } } tableName := strings.ToLower(string(table.Table)) if len(p.OnlyTables) > 0 { if db, ok := p.OnlyTables[tableName]; !ok { return false } else if db != "" && db != dbName { return false } } return true } // generateInsertSQL 生成insert语句 func (p *MyBinlogParser) generateInsertSQL(t *Table, e *replication.RowsEvent, binEvent *replication.BinlogEvent) error { tableName := getTableName(e) // if len(t.Columns) < int(e.ColumnCount) { // return fmt.Errorf("表%s缺少列!当前列数:%d,binlog的列数%d", // tableName, len(t.Columns), e.ColumnCount) // } var columnNames []string c := "`%s`" template := "INSERT INTO %s(%s) VALUES(%s)" if p.cfg.MinimalInsert && len(e.Rows) > 1 { template = "INSERT INTO %s(%s) VALUES%s" } for i, col := range t.Columns { if i < int(e.ColumnCount) && !col.IsGenerated() { // 有主键且设置去除主键时 作特殊处理 if t.hasPrimary && p.cfg.RemovePrimary { if _, ok := t.primarys[i]; !ok { columnNames = append(columnNames, fmt.Sprintf(c, col.ColumnName)) } } else { columnNames = append(columnNames, fmt.Sprintf(c, col.ColumnName)) } } } paramValues := strings.Repeat("?,", len(columnNames)) paramValues = strings.TrimRight(paramValues, ",") if p.cfg.MinimalInsert && len(e.Rows) > 1 { paramValues = strings.Repeat("("+paramValues+"),", len(e.Rows)) paramValues = strings.TrimRight(paramValues, ",") } sql := fmt.Sprintf(template, tableName, strings.Join(columnNames, ","), paramValues) var vv []driver.Value for _, rows := range e.Rows { for i, d := range rows { if t.Columns[i].IsGenerated() { continue } if t.hasPrimary && p.cfg.RemovePrimary { if _, ok := t.primarys[i]; ok { continue } } if t.Columns[i].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[i].ColumnType)) } vv = append(vv, d) } if !p.cfg.MinimalInsert || len(e.Rows) == 1 { r, err := InterpolateParams(sql, vv) if err != nil { log.Error(err) } p.write(r, binEvent) vv = nil } } if p.cfg.MinimalInsert && len(e.Rows) > 1 { r, err := InterpolateParams(sql, vv) if err != nil { log.Error(err) } p.write(r, binEvent) } return nil } func (p *MyBinlogParser) checkCanParse(t *Table, e *replication.RowsEvent) bool { if len(t.Columns) < int(e.ColumnCount) { tableName := getTableName(e) if _, ok := p.ignoreTables[tableName]; !ok { p.ignoreTables[tableName] = nil log.Warn(fmt.Sprintf("表%s缺少列,已忽略.(当前列数:%d,binlog的列数%d)", tableName, len(t.Columns), e.ColumnCount)) } return false } return true } // generateDeleteSQL 生成delete语句 func (p *MyBinlogParser) generateDeleteSQL(t *Table, e *replication.RowsEvent, binEvent *replication.BinlogEvent) error { tableName := getTableName(e) // if len(t.Columns) < int(e.ColumnCount) { // return fmt.Errorf("表%s缺少列!当前列数:%d,binlog的列数%d", // tableName, len(t.Columns), e.ColumnCount) // } template := "DELETE FROM %s WHERE" sql := fmt.Sprintf(template, tableName) c_null := " `%s` IS ?" c := " `%s`=?" var columnNames []string for _, rows := range e.Rows { columnNames = nil var vv []driver.Value for i, d := range rows { if t.hasPrimary { _, ok := t.primarys[i] if ok { if t.Columns[i].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[i].ColumnType)) } vv = append(vv, d) if d == nil { columnNames = append(columnNames, fmt.Sprintf(c_null, t.Columns[i].ColumnName)) } else { columnNames = append(columnNames, fmt.Sprintf(c, t.Columns[i].ColumnName)) } } } else { if t.Columns[i].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[i].ColumnType)) } vv = append(vv, d) if d == nil { columnNames = append(columnNames, fmt.Sprintf(c_null, t.Columns[i].ColumnName)) } else { columnNames = append(columnNames, fmt.Sprintf(c, t.Columns[i].ColumnName)) } } } newSql := strings.Join([]string{sql, strings.Join(columnNames, " AND")}, "") r, err := InterpolateParams(newSql, vv) if err != nil { log.Error(err) } p.write(r, binEvent) } return nil } // processValue 处理无符号值(unsigned) func processValue(value driver.Value, dataType string) driver.Value { if value == nil { return value } switch v := value.(type) { case int8: if v >= 0 { return value } return int64(1<<8 + int64(v)) case int16: if v >= 0 { return value } return int64(1<<16 + int64(v)) case int32: if v >= 0 { return value } if dataType == "mediumint" { return int64(1<<24 + int64(v)) } return int64(1<<32 + int64(v)) case int64: if v >= 0 { return value } return math.MaxUint64 - uint64(abs(v)) + 1 // case int: // case float32: // case float64: default: // log.Error("解析错误") // log.Errorf("%T", v) return value } } func abs(n int64) int64 { y := n >> 63 return (n ^ y) - y } // generateUpdateSQL 生成update语句 func (p *MyBinlogParser) generateUpdateSQL(t *Table, e *replication.RowsEvent, binEvent *replication.BinlogEvent) error { tableName := getTableName(e) // if len(t.Columns) < int(e.ColumnCount) { // return fmt.Errorf("表%s缺少列!当前列数:%d,binlog的列数%d", // tableName, len(t.Columns), e.ColumnCount) // } template := "UPDATE %s SET%s WHERE" setValue := " `%s`=?" var columnNames []string c_null := " `%s` IS ?" c := " `%s`=?" var sql string var sets []string // for i, col := range t.Columns { // if i < int(e.ColumnCount) { // sets = append(sets, fmt.Sprintf(setValue, col.ColumnName)) // } // } // 最小化回滚语句, 当开启时,update语句中未变更的值不再记录到回滚语句中 minimalMode := p.cfg.MinimalUpdate if !minimalMode { for i, col := range t.Columns { if i < int(e.ColumnCount) && !col.IsGenerated() { sets = append(sets, fmt.Sprintf(setValue, col.ColumnName)) } } sql = fmt.Sprintf(template, tableName, strings.Join(sets, ",")) } // sql := fmt.Sprintf(template,tableName, // strings.Join(sets, ",")) var ( oldValues []driver.Value newValues []driver.Value newSql string ) // update时, Rows为2的倍数, 双数index为旧值,单数index为新值 for i, rows := range e.Rows { if i%2 == 0 { // 旧值 columnNames = nil for j, d := range rows { if t.Columns[j].IsGenerated() { continue } if t.hasPrimary { if _, ok := t.primarys[j]; ok { oldValues = append(oldValues, d) if d == nil { columnNames = append(columnNames, fmt.Sprintf(c_null, t.Columns[j].ColumnName)) } else { columnNames = append(columnNames, fmt.Sprintf(c, t.Columns[j].ColumnName)) } } } else { if t.Columns[j].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[j].ColumnType)) } oldValues = append(oldValues, d) if d == nil { columnNames = append(columnNames, fmt.Sprintf(c_null, t.Columns[j].ColumnName)) } else { columnNames = append(columnNames, fmt.Sprintf(c, t.Columns[j].ColumnName)) } } } } else { // 新值 for j, d := range rows { if t.Columns[j].IsGenerated() { continue } if minimalMode { // 最小化模式下,列如果相等则省略 if !compareValue(d, e.Rows[i-1][j]) { if t.Columns[j].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[j].ColumnType)) } newValues = append(newValues, d) if j < len(t.Columns) { sets = append(sets, fmt.Sprintf(setValue, t.Columns[j].ColumnName)) } } } else { if t.Columns[j].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[j].ColumnType)) } newValues = append(newValues, d) } } newValues = append(newValues, oldValues...) if minimalMode { sql = fmt.Sprintf(template, tableName, strings.Join(sets, ",")) sets = nil } newSql = strings.Join([]string{sql, strings.Join(columnNames, " AND")}, "") // log.Info(newSql, len(newValues)) r, err := InterpolateParams(newSql, newValues) p.checkError(err) p.write(r, binEvent) oldValues = nil newValues = nil } } return nil } // generateUpdateRollbackSQL 生成update语句 func (p *MyBinlogParser) generateUpdateRollbackSQL(t *Table, e *replication.RowsEvent, binEvent *replication.BinlogEvent) error { tableName := getTableName(e) // if len(t.Columns) < int(e.ColumnCount) { // return fmt.Errorf("表%s缺少列!当前列数:%d,binlog的列数%d", // tableName, len(t.Columns), e.ColumnCount) // } template := "UPDATE %s SET%s WHERE" setValue := " `%s`=?" var columnNames []string c_null := " `%s` IS ?" c := " `%s`=?" var sql string var sets []string // 最小化回滚语句, 当开启时,update语句中未变更的值不再记录到回滚语句中 minimalMode := p.cfg.MinimalUpdate if !minimalMode { for i, col := range t.Columns { if i < int(e.ColumnCount) && !col.IsGenerated() { sets = append(sets, fmt.Sprintf(setValue, col.ColumnName)) } } sql = fmt.Sprintf(template, tableName, strings.Join(sets, ",")) } var ( oldValues []driver.Value newValues []driver.Value newSql string ) // update时, Rows为2的倍数, 双数index为旧值,单数index为新值 for i, rows := range e.Rows { if i%2 == 0 { // 旧值 for j, d := range rows { if t.Columns[j].IsGenerated() { continue } if minimalMode { // 最小化模式下,列如果相等则省略 if !compareValue(d, e.Rows[i+1][j]) { if t.Columns[j].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[j].ColumnType)) } newValues = append(newValues, d) if j < len(t.Columns) { sets = append(sets, fmt.Sprintf(setValue, t.Columns[j].ColumnName)) } } } else { if t.Columns[j].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[j].ColumnType)) } newValues = append(newValues, d) } } } else { // 新值 if p.cfg.Flashback { columnNames = nil } for j, d := range rows { if t.hasPrimary { if _, ok := t.primarys[j]; ok { if t.Columns[j].IsGenerated() { continue } if t.Columns[j].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[j].ColumnType)) } oldValues = append(oldValues, d) if d == nil { columnNames = append(columnNames, fmt.Sprintf(c_null, t.Columns[j].ColumnName)) } else { columnNames = append(columnNames, fmt.Sprintf(c, t.Columns[j].ColumnName)) } } } else { if t.Columns[j].IsGenerated() { continue } if t.Columns[j].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[j].ColumnType)) } oldValues = append(oldValues, d) if d == nil { columnNames = append(columnNames, fmt.Sprintf(c_null, t.Columns[j].ColumnName)) } else { columnNames = append(columnNames, fmt.Sprintf(c, t.Columns[j].ColumnName)) } } } newValues = append(newValues, oldValues...) if minimalMode { sql = fmt.Sprintf(template, tableName, strings.Join(sets, ",")) sets = nil } newSql = strings.Join([]string{sql, strings.Join(columnNames, " AND")}, "") r, err := InterpolateParams(newSql, newValues) p.checkError(err) p.write(r, binEvent) oldValues = nil newValues = nil } } return nil } func (p *MyBinlogParser) tableInformation(tableId uint64, schema []byte, tableName []byte) (*Table, error) { table, ok := p.allTables[tableId] if ok { return table, nil } if p.localMode && len(p.tableCacheList) > 0 { key := strings.ToLower(fmt.Sprintf("%s.%s", string(schema), string(tableName))) if table, ok := p.tableCacheList[key]; ok { p.allTables[tableId] = table return table, nil } else { key = strings.ToLower(string(tableName)) if table, ok := p.tableCacheList[key]; ok { p.allTables[tableId] = table return table, nil } } return nil, nil } sql := `SELECT COLUMN_NAME, ifnull(COLLATION_NAME,'') as COLLATION_NAME, ifnull(CHARACTER_SET_NAME,'') as CHARACTER_SET_NAME, ifnull(COLUMN_COMMENT,'') as COLUMN_COMMENT, COLUMN_TYPE, ifnull(COLUMN_KEY,'') as COLUMN_KEY, ifnull(EXTRA,'') as EXTRA FROM information_schema.columns WHERE table_schema = ? and table_name = ? ORDER BY ORDINAL_POSITION` var columns []Column p.db.Raw(sql, string(schema), string(tableName)).Scan(&columns) primarys := make(map[int]bool) uniques := make(map[int]bool) for i, r := range columns { if r.ColumnKey == "PRI" { primarys[i] = true } if r.ColumnKey == "UNI" { uniques[i] = true } } newTable := new(Table) newTable.tableID = tableId newTable.Columns = columns // if !p.cfg.AllColumns { if len(primarys) > 0 { newTable.primarys = primarys newTable.hasPrimary = true } else if len(uniques) > 0 { newTable.primarys = uniques newTable.hasPrimary = true } else { newTable.hasPrimary = false } // } p.allTables[tableId] = newTable return newTable, nil } func (p *MyBinlogParser) mysqlMasterStatus() (*MasterStatus, error) { defer timeTrack(time.Now(), "mysqlMasterStatus") // rows, err := db.Query("SHOW MASTER STATUS") // p.checkError(err) // defer rows.Close() r := MasterStatus{} p.db.Raw("SHOW MASTER STATUS").Scan(&r) // columns, _ := rows.Columns() // r := MasterStatus{} // for rows.Next() { // if len(columns) == 4 { // err = rows.Scan(&(r.File), &(r.Position), &(r.Binlog_Do_DB), // &(r.Binlog_Ignore_DB)) // } else if len(columns) == 5 { // err = rows.Scan(&(r.File), &(r.Position), &(r.Binlog_Do_DB), // &(r.Binlog_Ignore_DB), &(r.Executed_Gtid_Set)) // } else { // return nil, errors.New("show master status返回的列数无法解析!!!") // } // p.checkError(err) // } return &r, nil } func (p *MyBinlogParser) autoParseBinlogPosition() []MasterLog { if p.binlogs != nil { return p.binlogs } var binlogIndex []MasterLog p.db.Raw("SHOW MASTER LOGS").Scan(&binlogIndex) p.binlogs = binlogIndex return binlogIndex } // InterpolateParams 填充占位符参数 func InterpolateParams(query string, args []driver.Value) ([]byte, error) { // Number of ? should be same to len(args) if strings.Count(query, "?") != len(args) { log.WithFields(log.Fields{ "需要参数": strings.Count(query, "?"), "提供参数": len(args), }).Error("sql的参数个数不匹配") return nil, errors.New("driver: skip fast-path; continue as if unimplemented") } var buf []byte argPos := 0 for i := 0; i < len(query); i++ { q := strings.IndexByte(query[i:], '?') if q == -1 { buf = append(buf, query[i:]...) break } buf = append(buf, query[i:i+q]...) i += q arg := args[argPos] argPos++ if arg == nil { buf = append(buf, "NULL"...) continue } switch v := arg.(type) { case int8: buf = strconv.AppendInt(buf, int64(v), 10) case int16: buf = strconv.AppendInt(buf, int64(v), 10) case int32: buf = strconv.AppendInt(buf, int64(v), 10) case int64: buf = strconv.AppendInt(buf, v, 10) case uint64: buf = strconv.AppendUint(buf, uint64(v), 10) case int: buf = strconv.AppendInt(buf, int64(v), 10) case decimal.Decimal: buf = append(buf, v.String()...) case float32: buf = strconv.AppendFloat(buf, float64(v), 'g', -1, 32) case float64: buf = strconv.AppendFloat(buf, v, 'g', -1, 64) case bool: if v { buf = append(buf, '1') } else { buf = append(buf, '0') } case time.Time: if v.IsZero() { buf = append(buf, "'0000-00-00'"...) } else { v := v.In(time.UTC) v = v.Add(time.Nanosecond * 500) // To round under microsecond year := v.Year() year100 := year / 100 year1 := year % 100 month := v.Month() day := v.Day() hour := v.Hour() minute := v.Minute() second := v.Second() micro := v.Nanosecond() / 1000 buf = append(buf, []byte{ '\'', digits10[year100], digits01[year100], digits10[year1], digits01[year1], '-', digits10[month], digits01[month], '-', digits10[day], digits01[day], ' ', digits10[hour], digits01[hour], ':', digits10[minute], digits01[minute], ':', digits10[second], digits01[second], }...) if micro != 0 { micro10000 := micro / 10000 micro100 := micro / 100 % 100 micro1 := micro % 100 buf = append(buf, []byte{ '.', digits10[micro10000], digits01[micro10000], digits10[micro100], digits01[micro100], digits10[micro1], digits01[micro1], }...) } buf = append(buf, '\'') } case string: buf = append(buf, '\'') buf = escapeBytesBackslash(buf, []byte(v)) buf = append(buf, '\'') case []byte: if v == nil { buf = append(buf, "NULL"...) } else { // buf = append(buf, "_binary'"...) buf = append(buf, '\'') buf = escapeBytesBackslash(buf, v) buf = append(buf, '\'') } default: // fmt.Println(v) log.Printf("%T", v) log.Info("解析错误") return nil, errors.New("driver: skip fast-path; continue as if unimplemented") } // 4 << 20 , 4MB // 移除对单行大小的判断,不入库不需要该限制 // if len(buf)+4 > 4<<20 { // // log.Print("%T", v) // log.Info("解析错误") // return nil, errors.New("driver: skip fast-path; continue as if unimplemented") // } } if argPos != len(args) { // log.Print("%T", v) log.Info("解析错误") return nil, errors.New("driver: skip fast-path; continue as if unimplemented") } return buf, nil } // reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize. // If cap(buf) is not enough, reallocate new buffer. func reserveBuffer(buf []byte, appendSize int) []byte { newSize := len(buf) + appendSize if cap(buf) < newSize { // Grow buffer exponentially newBuf := make([]byte, len(buf)*2+appendSize) copy(newBuf, buf) buf = newBuf } return buf[:newSize] } // escapeBytesBackslash escapes []byte with backslashes (\) // This escapes the contents of a string (provided as []byte) by adding backslashes before special // characters, and turning others into specific escape sequences, such as // turning newlines into \n and null bytes into \0. // https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L823-L932 func escapeBytesBackslash(buf, v []byte) []byte { pos := len(buf) buf = reserveBuffer(buf, len(v)*2) for _, c := range v { switch c { case '\x00': buf[pos] = '\\' buf[pos+1] = '0' pos += 2 case '\n': buf[pos] = '\\' buf[pos+1] = 'n' pos += 2 case '\r': buf[pos] = '\\' buf[pos+1] = 'r' pos += 2 case '\x1a': buf[pos] = '\\' buf[pos+1] = 'Z' pos += 2 case '\'': buf[pos] = '\\' buf[pos+1] = '\'' pos += 2 case '"': buf[pos] = '\\' buf[pos+1] = '"' pos += 2 case '\\': buf[pos] = '\\' buf[pos+1] = '\\' pos += 2 default: buf[pos] = c pos++ } } return buf[:pos] } // escapeStringBackslash is similar to escapeBytesBackslash but for string. func escapeStringBackslash(buf []byte, v string) []byte { pos := len(buf) buf = reserveBuffer(buf, len(v)*2) for i := 0; i < len(v); i++ { c := v[i] switch c { case '\x00': buf[pos] = '\\' buf[pos+1] = '0' pos += 2 case '\n': buf[pos] = '\\' buf[pos+1] = 'n' pos += 2 case '\r': buf[pos] = '\\' buf[pos+1] = 'r' pos += 2 case '\x1a': buf[pos] = '\\' buf[pos+1] = 'Z' pos += 2 case '\'': buf[pos] = '\\' buf[pos+1] = '\'' pos += 2 case '"': buf[pos] = '\\' buf[pos+1] = '"' pos += 2 case '\\': buf[pos] = '\\' buf[pos+1] = '\\' pos += 2 default: buf[pos] = c pos++ } } return buf[:pos] } // escapeBytesQuotes escapes apostrophes in []byte by doubling them up. // This escapes the contents of a string by doubling up any apostrophes that // it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in // effect on the server. // https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L963-L1038 func escapeBytesQuotes(buf, v []byte) []byte { pos := len(buf) buf = reserveBuffer(buf, len(v)*2) for _, c := range v { if c == '\'' { buf[pos] = '\'' buf[pos+1] = '\'' pos += 2 } else { buf[pos] = c pos++ } } return buf[:pos] } // escapeStringQuotes is similar to escapeBytesQuotes but for string. func escapeStringQuotes(buf []byte, v string) []byte { pos := len(buf) buf = reserveBuffer(buf, len(v)*2) for i := 0; i < len(v); i++ { c := v[i] if c == '\'' { buf[pos] = '\'' buf[pos+1] = '\'' pos += 2 } else { buf[pos] = c pos++ } } return buf[:pos] } // GetDataTypeBase 获取dataType中的数据类型，忽略长度 func GetDataTypeBase(dataType string) string { if i := strings.Index(dataType, "("); i > 0 { return dataType[0:i] } return dataType } // readTableSchema 读取表结构 func (p *MyBinlogParser) readTableSchema(path string) error { fileObj, err := os.Open(path) if err != nil { return err } defer fileObj.Close() reader := bufio.NewReader(fileObj) p.tableCacheList = make(map[string]*Table) var buf []string quotaIsDouble := true parser := tidbParser.New() for { line, err := reader.ReadString('\n') if err != nil { if err != io.EOF { log.Println(err) break } } if strings.Count(line, "'")%2 == 1 { quotaIsDouble = !quotaIsDouble } if ((strings.HasSuffix(line, ";") || strings.HasSuffix(line, ";\r")) && quotaIsDouble) || err == io.EOF { buf = append(buf, line) s1 := strings.Join(buf, "\n") s1 = strings.TrimRight(s1, ";") buf = nil stmtNodes, _, err := parser.Parse(s1, "utf8mb4", "utf8mb4_bin") if err != nil { return fmt.Errorf("解析失败: %v", err) } for _, stmtNode := range stmtNodes { //  是ASCII码160的特殊空格 // currentSql := strings.Trim(stmtNode.Text(), " ;\t\n\v\f\r ") switch node := stmtNode.(type) { case *ast.CreateTableStmt: p.cacheNewTable(buildTableInfo(node)) } } } else { buf = append(buf, line) } if err == io.EOF { break } } return nil } // cacheNewTable 缓存表结构 func (p *MyBinlogParser) cacheNewTable(t *Table) { var key string if t.Schema == "" { key = t.TableName } else { key = fmt.Sprintf("%s.%s", t.Schema, t.TableName) } key = strings.ToLower(key) p.tableCacheList[key] = t p.OnlyTables[strings.ToLower(t.TableName)] = t.Schema } // buildTableInfo 构建表结构 func buildTableInfo(node *ast.CreateTableStmt) *Table { table := &Table{ Schema: node.Table.Schema.String(), TableName: node.Table.Name.String(), } for _, ct := range node.Constraints { switch ct.Tp { // 设置主键标志 case ast.ConstraintPrimaryKey: for _, col := range ct.Keys { for _, field := range node.Cols { if field.Name.Name.L == col.Column.Name.L { field.Tp.Flag |= tidb.PriKeyFlag break } } } case ast.ConstraintUniq, ast.ConstraintUniqIndex, ast.ConstraintUniqKey: for _, col := range ct.Keys { for _, field := range node.Cols { if field.Name.Name.L == col.Column.Name.L { field.Tp.Flag |= tidb.UniqueKeyFlag break } } } } } table.Columns = make([]Column, 0, len(node.Cols)) for _, field := range node.Cols { table.Columns = append(table.Columns, *buildNewColumnToCache(table, field)) } table.configPrimaryKey() return table } // buildNewColumnToCache 构建列 func buildNewColumnToCache(t *Table, field *ast.ColumnDef) *Column { c := &Column{} c.ColumnName = field.Name.Name.String() c.ColumnType = field.Tp.InfoSchemaStr() for _, op := range field.Options { switch op.Tp { case ast.ColumnOptionComment: c.ColumnComment = op.Expr.GetDatum().GetString() case ast.ColumnOptionPrimaryKey: c.ColumnKey = "PRI" case ast.ColumnOptionUniqKey: c.ColumnKey = "UNI" case ast.ColumnOptionAutoIncrement: // c.Extra += "auto_increment" case ast.ColumnOptionGenerated: if op.Stored { c.Extra += " STORED GENERATED" } else { c.Extra += " VIRTUAL GENERATED" } } } if c.ColumnKey != "PRI" && tidb.HasPriKeyFlag(field.Tp.Flag) { c.ColumnKey = "PRI" } else if tidb.HasUniKeyFlag(field.Tp.Flag) { c.ColumnKey = "UNI" } return c } // compareValue 比较两值是否相等 func compareValue(v1 interface{}, v2 interface{}) bool { equal := false // 处理特殊情况 if v1 == nil && v2 == nil { return true } if v1 == nil || v2 == nil { return false } switch v := v1.(type) { case []byte: equal = byteEquals(v, v2.([]byte)) case decimal.Decimal: if newDec, ok := v2.(decimal.Decimal); ok { equal = v.Equal(newDec) } else { equal = false } default: equal = v1 == v2 } return equal } // configPrimaryKey 配置主键设置 func (t *Table) configPrimaryKey() { var primarys map[int]bool primarys = make(map[int]bool) var uniques map[int]bool uniques = make(map[int]bool) for i, r := range t.Columns { if r.ColumnKey == "PRI" { primarys[i] = true } if r.ColumnKey == "UNI" { uniques[i] = true } } // newTable.tableId = tableId if len(primarys) > 0 { t.primarys = primarys t.hasPrimary = true } else if len(uniques) > 0 { t.primarys = uniques t.hasPrimary = true } else { t.hasPrimary = false } } func (p *MyBinlogParser) parseSingleEvent(e *replication.BinlogEvent) (ok bool, err error) { // 是否继续,默认为true ok = true finishFlag := -1 if e.Header.LogPos > 0 { p.currentPosition.Pos = e.Header.LogPos } if e.Header.EventType == replication.ROTATE_EVENT { if event, ok := e.Event.(*replication.RotateEvent); ok { p.currentPosition = mysql.Position{ Name: string(event.NextLogName), Pos: uint32(event.Position)} } } if !p.cfg.StopNever { if e.Header.Timestamp > 0 { if p.startTimestamp > 0 && e.Header.Timestamp < p.startTimestamp { return } if p.stopTimestamp > 0 && e.Header.Timestamp > p.stopTimestamp { log.Warn("已超出结束时间") return false, nil } } p.currentTimtstamp = e.Header.Timestamp finishFlag = p.checkFinish(&p.currentPosition) if finishFlag == 1 { log.Warn("is finish") return false, nil } } // 只需解析GTID返回内的event if len(p.includeGtids) > 0 { switch e.Header.EventType { case replication.TABLE_MAP_EVENT, replication.QUERY_EVENT, replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2, replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2, replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2: status := p.isGtidEventInGtidSet() if status == 1 { return } else if status == 2 { log.Info("已超出GTID范围,自动结束") if !p.cfg.StopNever { return false, nil } } } } switch event := e.Event.(type) { case *replication.GTIDEvent: if len(p.includeGtids) > 0 { p.gtidEvent = event } // e.Dump(os.Stdout) u, _ := uuid.FromBytes(event.SID) p.gtid = append([]byte(u.String()), []byte(fmt.Sprintf(":%d", event.GNO))...) // fmt.Println(p.gtid) case *replication.TableMapEvent: if !p.schemaFilter(event) { return } _, err = p.tableInformation(event.TableID, event.Schema, event.Table) if err != nil { return false, err } case *replication.QueryEvent: if p.cfg.ThreadID > 0 || p.cfg.ShowThread { p.currentThreadID = event.SlaveProxyID if p.cfg.ThreadID != p.currentThreadID { return } } // 回滚或者仅dml语句时 跳过 if p.cfg.Flashback || !p.cfg.ParseDDL { return } if string(event.Query) != "BEGIN" && string(event.Query) != "COMMIT" { if len(event.Schema) > 0 { p.write(append([]byte(fmt.Sprintf("USE `%s`;\n", event.Schema)), event.Query...), e) } // changeRows++ } else { // fmt.Println(string(event.Query)) // log.Info("#start %d %d %d", e.Header.LogPos, // e.Header.LogPos+e.Header.EventSize, // e.Header.LogPos-e.Header.EventSize) // if binlog_event.query == 'BEGIN': // e_start_pos = last_pos } case *replication.RowsEvent: if !p.schemaFilter(event.Table) { return } if p.cfg.ThreadID > 0 && p.cfg.ThreadID != p.currentThreadID { return } table, err := p.tableInformation(event.TableID, event.Table.Schema, event.Table.Table) if err != nil { return false, err } switch e.Header.EventType { case replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2: if _, ok := p.SqlTypes["insert"]; ok && p.checkCanParse(table, event) { if p.cfg.Flashback { err = p.generateDeleteSQL(table, event, e) } else { err = p.generateInsertSQL(table, event, e) } p.changeRows = p.changeRows + len(event.Rows) } case replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2: if _, ok := p.SqlTypes["delete"]; ok && p.checkCanParse(table, event) { if p.cfg.Flashback { err = p.generateInsertSQL(table, event, e) } else { err = p.generateDeleteSQL(table, event, e) } p.changeRows = p.changeRows + len(event.Rows) } case replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2: if _, ok := p.SqlTypes["update"]; ok && p.checkCanParse(table, event) { if p.cfg.Flashback { err = p.generateUpdateRollbackSQL(table, event, e) } else { err = p.generateUpdateSQL(table, event, e) } p.changeRows = p.changeRows + len(event.Rows)/2 } } if err != nil { return false, err } } if p.cfg.MaxRows > 0 && p.changeRows >= p.cfg.MaxRows { log.Info("已超出最大行数") return false, nil } // 再次判断是否到结束位置,以免处在临界值,且无新日志时程序卡住 if e.Header.Timestamp > 0 { if p.stopTimestamp > 0 && e.Header.Timestamp > p.stopTimestamp { log.Warn("已超出结束时间") return false, nil } } if !p.running { log.Warn("进程手动中止") return false, nil } if finishFlag > -1 { return false, nil } return } // ParseRows 获取已解析行数 func (p *MyBinlogParser) Config() *BinlogParserConfig { return p.cfg } // ParseRows 获取已解析行数 func (p *MyBinlogParser) ParseRows() int { return p.changeRows } // Percent 获取解析百分比 func (p *MyBinlogParser) Percent() int { if !p.running { return 100 } if p.cfg.StartFile != "" { start := mysql.Position{ Name: p.cfg.StartFile, Pos: uint32(p.cfg.StartPosition), } stop := mysql.Position{ Name: p.cfg.StopFile, Pos: uint32(p.cfg.StopPosition), } return ComputePercent(start, stop, p.currentPosition, *p.masterStatus, p.autoParseBinlogPosition()) } if p.stopTimestamp > 0 { if p.currentTimtstamp < p.startTimestamp || p.stopTimestamp == p.startTimestamp { return 0 } else { return int((p.currentTimtstamp - p.startTimestamp) * 100 / (p.stopTimestamp - p.startTimestamp)) } } if p.cfg.MaxRows > 0 { if p.changeRows < p.cfg.MaxRows { return p.changeRows / p.cfg.MaxRows * 100 } return 100 } return 0 } // Archive文件压缩打包 func (p *MyBinlogParser) Archive() (fileSize int64, err error) { var url string if strings.Count(p.outputFileName, ".") > 0 { a := strings.Split(p.outputFileName, ".") url = strings.Join(a[0:len(a)-1], ".") } else { url = p.outputFileName } url = url + ".tar.gz" err = archiver.Archive([]string{p.outputFileName}, url) if err != nil { return 0, err } fileInfo, _ := os.Stat(url) //文件大小 fileSize = fileInfo.Size() // 压缩完成后删除文件 p.clear() log.Info("打包完成") return fileSize, nil } // getTableName 获取RowsEvent的表名 func getTableName(e *replication.RowsEvent) string { var build strings.Builder build.WriteString("`") build.Write(e.Table.Schema) build.WriteString("`.`") build.Write(e.Table.Table) build.WriteString("`") return build.String() } func ComputePercent(start, stop, currnet mysql.Position, masterStatus MasterStatus, binlogs []MasterLog) int { if start.Name == stop.Name && stop.Pos > 0 { return int((currnet.Pos - start.Pos) * 100 / (stop.Pos - start.Pos)) } else { stopFile := stop.Name stopPosition := int(stop.Pos) if stopFile == "" { stopFile = masterStatus.File stopPosition = masterStatus.Position } total := 0 parsed := 0 for _, binlog := range binlogs { if binlog.Name < start.Name { continue } if binlog.Name > stopFile { break } // log.Info("---") // log.Infof("total: %d, parsed: %d, binlog: %s %v", // total, parsed, binlog.Name, binlog.Size) // 第一个日志文件指定了文件 if binlog.Name == start.Name && binlog.Size > int(start.Pos) { total += binlog.Size - int(start.Pos) } else if binlog.Name == stopFile { // 最后一个日志文件 if stopPosition > 0 { total += stopPosition } else { total += binlog.Size } } else if binlog.Name < stopFile { total += binlog.Size } if currnet.Name == binlog.Name { if binlog.Name == start.Name { if currnet.Pos > start.Pos { parsed += int(currnet.Pos - start.Pos) } } else { parsed += int(currnet.Pos) } } else if binlog.Name < currnet.Name { if binlog.Name == start.Name { if binlog.Size > int(start.Pos) { parsed += binlog.Size - int(start.Pos) } } else { parsed += binlog.Size } } // log.Infof("total: %d, parsed: %d, binlog: %s %v", // total, parsed, binlog.Name, binlog.Size) } pecent := 0 if total > 0 { pecent = parsed * 100 / total } if parsed >= total { pecent = 100 } // log.Infof("parsed: %d, total: %d, percent: %v", // parsed, total, pecent) return pecent } } ================================================ FILE: core/parserV2.go ================================================ package core import ( "bytes" "database/sql/driver" "errors" "fmt" "strconv" "strings" "time" "github.com/go-mysql-org/go-mysql/replication" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" ) // generateInsertSQL 生成insert语句 func (p *MyBinlogParser) generateInsertSQL_2(t *Table, e *replication.RowsEvent, binEvent *replication.BinlogEvent) error { if len(t.Columns) < int(e.ColumnCount) { return fmt.Errorf("表%s.%s缺少列!当前列数:%d,binlog的列数%d", e.Table.Schema, e.Table.Table, len(t.Columns), e.ColumnCount) } var columnNames []string c := "`%s`" buf := &bytes.Buffer{} buf.WriteString("INSERT ") // if s.ignore { // _, _ = buf.WriteString("IGNORE ") // } buf.WriteString("INTO ") buf.WriteString("`") buf.Write(e.Table.Schema) buf.WriteString("`.`") buf.Write(e.Table.Table) buf.WriteString("`") buf.WriteString("(") strings.Join(columnNames, ",") for i, col := range t.Columns { if i < int(e.ColumnCount) { // 有主键且设置去除主键时 作特殊处理 if t.hasPrimary && p.cfg.RemovePrimary { if _, ok := t.primarys[i]; !ok { columnNames = append(columnNames, fmt.Sprintf(c, col.ColumnName)) } } else { columnNames = append(columnNames, fmt.Sprintf(c, col.ColumnName)) } } } buf.WriteString(strings.Join(columnNames, ",")) buf.WriteString(") VALUES") var insertSQL string if !p.Config().MinimalInsert { insertSQL = buf.String() } for rowIndex, rows := range e.Rows { if rowIndex > 0 { if p.Config().MinimalInsert { buf.WriteString(",") } else { buf.WriteString(insertSQL) } } buf.WriteString("(") for colIndex, d := range rows { if t.hasPrimary && p.cfg.RemovePrimary { if _, ok := t.primarys[colIndex]; ok { continue } } if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } v, err := valueSerialize(d) if err != nil { log.Error(err) return err } if colIndex > 0 { buf.WriteByte(',') } buf.Write(v) } buf.WriteString(")") if !p.cfg.MinimalInsert { p.write(buf.Bytes(), binEvent) buf = &bytes.Buffer{} } } if p.cfg.MinimalInsert { p.write(buf.Bytes(), binEvent) buf.Reset() } return nil } // generateDeleteSQL 生成delete语句 func (p *MyBinlogParser) generateDeleteSQL_2(t *Table, e *replication.RowsEvent, binEvent *replication.BinlogEvent) error { if len(t.Columns) < int(e.ColumnCount) { return fmt.Errorf("表%s.%s缺少列!当前列数:%d,binlog的列数%d", e.Table.Schema, e.Table.Table, len(t.Columns), e.ColumnCount) } for _, rows := range e.Rows { buf := &bytes.Buffer{} buf.WriteString("DELETE FROM ") buf.WriteString("`") buf.Write(e.Table.Schema) buf.WriteString("`.`") buf.Write(e.Table.Table) buf.WriteString("` WHERE ") initFirst := false for colIndex, d := range rows { if t.hasPrimary { if _, ok := t.primarys[colIndex]; ok { if initFirst { buf.WriteString(" AND ") } initFirst = true if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } if d == nil { buf.WriteString("`") buf.WriteString(t.Columns[colIndex].ColumnName) buf.WriteString("` IS NULL") } else { v, err := valueSerialize(d) if err != nil { log.Error(err) } buf.WriteString("`") buf.WriteString(t.Columns[colIndex].ColumnName) buf.WriteString("`=") buf.Write(v) } } } else { if initFirst { buf.WriteString(" AND ") } initFirst = true if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } if d == nil { buf.WriteString("`") buf.WriteString(t.Columns[colIndex].ColumnName) buf.WriteString("` IS NULL") } else { v, err := valueSerialize(d) if err != nil { log.Error(err) } buf.WriteString("`") buf.WriteString(t.Columns[colIndex].ColumnName) buf.WriteString("`=") buf.Write(v) } } } p.write(buf.Bytes(), binEvent) // buf = &bytes.Buffer{} } return nil } // generateUpdateSQL 生成update语句 func (p *MyBinlogParser) generateUpdateSQL_2(t *Table, e *replication.RowsEvent, binEvent *replication.BinlogEvent) error { if len(t.Columns) < int(e.ColumnCount) { return fmt.Errorf("表%s.%s缺少列!当前列数:%d,binlog的列数%d", e.Table.Schema, e.Table.Table, len(t.Columns), e.ColumnCount) } // 最小化回滚语句, 当开启时,update语句中未变更的值不再记录到回滚语句中 minimalMode := p.cfg.MinimalUpdate // update时, Rows为2的倍数, 双数index为旧值,单数index为新值 buf := &bytes.Buffer{} bufWhere := &bytes.Buffer{} for rowIndex, row := range e.Rows { if rowIndex%2 == 0 { // 旧值 for colIndex, d := range row { if t.hasPrimary { if _, ok := t.primarys[colIndex]; ok { if bufWhere.Len() > 0 { bufWhere.WriteString(" AND ") } if d == nil { bufWhere.WriteString("`") bufWhere.WriteString(t.Columns[colIndex].ColumnName) bufWhere.WriteString("` IS NULL") } else { bufWhere.WriteString("`") bufWhere.WriteString(t.Columns[colIndex].ColumnName) bufWhere.WriteString("`=") v, err := valueSerialize(d) if err != nil { log.Error(err) } bufWhere.Write(v) } } } else { if colIndex > 0 { bufWhere.WriteString(" AND ") } if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } if d == nil { bufWhere.WriteString("`") bufWhere.WriteString(t.Columns[colIndex].ColumnName) bufWhere.WriteString("` IS NULL") } else { v, err := valueSerialize(d) if err != nil { log.Error(err) } bufWhere.WriteString("`") bufWhere.WriteString(t.Columns[colIndex].ColumnName) bufWhere.WriteString("`=") bufWhere.Write(v) } } } } else { initFirst := false buf.WriteString("UPDATE ") buf.WriteString("`") buf.Write(e.Table.Schema) buf.WriteString("`.`") buf.Write(e.Table.Table) buf.WriteString("` SET ") for colIndex, d := range row { if minimalMode { // 最小化模式下,列如果相等则省略 if !compareValue(d, e.Rows[rowIndex-1][colIndex]) { if initFirst { buf.WriteByte(',') } initFirst = true if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } v, err := valueSerialize(d) if err != nil { log.Error(err) } buf.WriteString("`") buf.WriteString(t.Columns[colIndex].ColumnName) buf.WriteString("`=") buf.Write(v) } } else { if colIndex > 0 { buf.WriteByte(',') } if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } v, err := valueSerialize(d) if err != nil { log.Error(err) } buf.WriteString("`") buf.WriteString(t.Columns[colIndex].ColumnName) buf.WriteString("`=") buf.Write(v) } } buf.WriteString(" WHERE ") buf.Write(bufWhere.Bytes()) p.write(buf.Bytes(), binEvent) buf = &bytes.Buffer{} bufWhere = &bytes.Buffer{} } } return nil } // generateUpdateRollbackSQL 生成update语句 func (p *MyBinlogParser) generateUpdateRollbackSQL_2(t *Table, e *replication.RowsEvent, binEvent *replication.BinlogEvent) error { if len(t.Columns) < int(e.ColumnCount) { return fmt.Errorf("表%s.%s缺少列!当前列数:%d,binlog的列数%d", e.Table.Schema, e.Table.Table, len(t.Columns), e.ColumnCount) } // 最小化回滚语句, 当开启时,update语句中未变更的值不再记录到回滚语句中 minimalMode := p.cfg.MinimalUpdate // update时, Rows为2的倍数, 双数index为旧值,单数index为新值 buf := &bytes.Buffer{} bufWhere := &bytes.Buffer{} for rowIndex, rows := range e.Rows { // 旧值 if rowIndex%2 == 0 { buf.WriteString("UPDATE ") buf.WriteString("`") buf.Write(e.Table.Schema) buf.WriteString("`.`") buf.Write(e.Table.Table) buf.WriteString("` SET ") initFirst := false for colIndex, d := range rows { if minimalMode { // 最小化模式下,列如果相等则省略 if !compareValue(d, e.Rows[rowIndex+1][colIndex]) { if initFirst { buf.WriteString(",") } initFirst = true if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } v, err := valueSerialize(d) if err != nil { log.Error(err) } buf.WriteString("`") buf.WriteString(t.Columns[colIndex].ColumnName) buf.WriteString("`=") buf.Write(v) } } else { if colIndex > 0 { buf.WriteString(",") } if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } v, err := valueSerialize(d) if err != nil { log.Error(err) } buf.WriteString("`") buf.WriteString(t.Columns[colIndex].ColumnName) buf.WriteString("`=") buf.Write(v) } } } else { // 新值 for colIndex, d := range rows { if t.hasPrimary { _, ok := t.primarys[colIndex] if ok { if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } if bufWhere.Len() > 0 { bufWhere.WriteString(" AND ") } if d == nil { bufWhere.WriteString("`") bufWhere.WriteString(t.Columns[colIndex].ColumnName) bufWhere.WriteString("` IS NULL") } else { bufWhere.WriteString("`") bufWhere.WriteString(t.Columns[colIndex].ColumnName) bufWhere.WriteString("`=") v, err := valueSerialize(d) if err != nil { log.Error(err) } bufWhere.Write(v) } } } else { if t.Columns[colIndex].IsUnsigned() { d = processValue(d, GetDataTypeBase(t.Columns[colIndex].ColumnType)) } if bufWhere.Len() > 0 { bufWhere.WriteString(" AND ") } if d == nil { bufWhere.WriteString("`") bufWhere.WriteString(t.Columns[colIndex].ColumnName) bufWhere.WriteString("` IS NULL") } else { bufWhere.WriteString("`") bufWhere.WriteString(t.Columns[colIndex].ColumnName) bufWhere.WriteString("`=") v, err := valueSerialize(d) if err != nil { log.Error(err) } bufWhere.Write(v) } } } buf.WriteString(" WHERE ") buf.Write(bufWhere.Bytes()) p.write(buf.Bytes(), binEvent) buf = &bytes.Buffer{} bufWhere = &bytes.Buffer{} } } return nil } // valueSerialize 参数转换成数据库类型 func valueSerialize(arg driver.Value) (buf []byte, err error) { if arg == nil { buf = append(buf, "NULL"...) return buf, nil } switch v := arg.(type) { case int8: buf = strconv.AppendInt(buf, int64(v), 10) case int16: buf = strconv.AppendInt(buf, int64(v), 10) case int32: buf = strconv.AppendInt(buf, int64(v), 10) case int64: buf = strconv.AppendInt(buf, v, 10) case uint64: buf = strconv.AppendUint(buf, uint64(v), 10) case int: buf = strconv.AppendInt(buf, int64(v), 10) case decimal.Decimal: buf = append(buf, v.String()...) case float32: buf = strconv.AppendFloat(buf, float64(v), 'g', -1, 32) case float64: buf = strconv.AppendFloat(buf, v, 'g', -1, 64) case bool: if v { buf = append(buf, '1') } else { buf = append(buf, '0') } case time.Time: if v.IsZero() { buf = append(buf, "'0000-00-00'"...) } else { v := v.In(time.UTC) v = v.Add(time.Nanosecond * 500) // To round under microsecond year := v.Year() year100 := year / 100 year1 := year % 100 month := v.Month() day := v.Day() hour := v.Hour() minute := v.Minute() second := v.Second() micro := v.Nanosecond() / 1000 buf = append(buf, []byte{ '\'', digits10[year100], digits01[year100], digits10[year1], digits01[year1], '-', digits10[month], digits01[month], '-', digits10[day], digits01[day], ' ', digits10[hour], digits01[hour], ':', digits10[minute], digits01[minute], ':', digits10[second], digits01[second], }...) if micro != 0 { micro10000 := micro / 10000 micro100 := micro / 100 % 100 micro1 := micro % 100 buf = append(buf, []byte{ '.', digits10[micro10000], digits01[micro10000], digits10[micro100], digits01[micro100], digits10[micro1], digits01[micro1], }...) } buf = append(buf, '\'') } case string: buf = append(buf, '\'') buf = escapeBytesBackslash(buf, []byte(v)) buf = append(buf, '\'') case []byte: if v == nil { buf = append(buf, "NULL"...) } else { // buf = append(buf, "_binary'"...) buf = append(buf, '\'') buf = escapeBytesBackslash(buf, v) buf = append(buf, '\'') } default: // fmt.Println(v) log.Printf("%T", v) log.Info("解析错误") return nil, errors.New("driver: skip fast-path; continue as if unimplemented") } // 4 << 20 , 4MB if len(buf)+4 > 4<<20 { // log.Print("%T", v) log.Info("解析错误") return nil, errors.New("driver: skip fast-path; continue as if unimplemented") } return buf, nil } ================================================ FILE: core/parser_stats.go ================================================ package core import ( "bufio" "context" "errors" "fmt" "os" "sort" "strings" "time" "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" "github.com/jinzhu/now" log "github.com/sirupsen/logrus" ) type SummaryStats struct { Inserts int `json:"insert"` Updates int `json:"update"` Deletes int `json:"delete"` Total int `json:"total"` StartTime time.Time `json:"start_time"` EndTime time.Time `json:"end_time"` Tables []TableStats `json:"tables"` } type TableStats struct { Table string DB string Inserts int `json:"insert"` Updates int `json:"update"` Deletes int `json:"delete"` Total int `json:"total"` } func (s *SummaryStats) String() string { if len(s.Tables) > 1 { sort.SliceStable(s.Tables, func(cur, next int) bool { return s.Tables[cur].Total < s.Tables[next].Total }) } var tableStr string for _, t := range s.Tables { tableStr += t.String() } // fmt.Sprintf("start: %v\n"+" stop: %v\n", s.StartTime.Format(time.RFC3339), // s.EndTime.Format(time.RFC3339)) + "\n" + return tableStr + fmt.Sprintf("\n"+`Summary: %d insert: %d update: %d delete: %d`+"\n", s.Total, s.Inserts, s.Updates, s.Deletes) } func (t *TableStats) String() string { return fmt.Sprintf(`%s.%s: %d [insert:%d, update:%d, delete:%d]`+"\n", t.DB, t.Table, t.Total, t.Inserts, t.Updates, t.Deletes) } type BinlogParserStats struct { baseParser masterStatus *MasterStatus startTimestamp uint32 stopTimestamp uint32 // 输出到指定文件 outputFile *os.File outputFileName string // db *gorm.DB // 使用buffer写的测试 // 解析600M日志,约500w行,直接使用文件时,用时5min // 使用buffer用时1min bufferWriter *bufio.Writer // 表结构缓存(仅用于本地解析) tableCacheList map[string]*Table // 解析用临时变量 currentPosition mysql.Position // 当前binlog位置 currentTimtstamp uint32 // 当前解析到的时间戳 // 保存binlogs文件和position,用以计算percent binlogs []MasterLog stats SummaryStats statsTables map[string]*TableStats } // Parser 远程解析 func (p *BinlogParserStats) ParserStats() error { // runtime.GOMAXPROCS(runtime.NumCPU()) if p.cfg.Host == "" && p.cfg.StartFile != "" { _, err := os.Stat(p.cfg.StartFile) if err != nil { return fmt.Errorf("open start_file: %w", err) } return p.parserFile() } defer timeTrack(time.Now(), "Parser") defer func() { if p.outputFile != nil { p.outputFile.Close() } }() var err error p.running = true p.db, err = p.getDB() if err != nil { return err } defer p.db.Close() if len(p.outputFileName) > 0 { if Exists(p.outputFileName) { // p.checkError(errors.New("指定的文件已存在!")) // 不作追加,改为文件存在时清除内容,O_APPEND p.outputFile, err = os.OpenFile(p.outputFileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) } else { p.outputFile, err = os.Create(p.outputFileName) } if err != nil { return fmt.Errorf("create output file: %w", err) } p.bufferWriter = bufio.NewWriter(p.outputFile) } cfg := replication.BinlogSyncerConfig{ ServerID: 2000000211, Flavor: p.cfg.Flavor, Host: p.cfg.Host, Port: p.cfg.Port, User: p.cfg.User, Password: p.cfg.Password, UseDecimal: true, } b := replication.NewBinlogSyncer(cfg) defer b.Close() p.currentPosition = mysql.Position{ Name: p.startFile, Pos: uint32(p.cfg.StartPosition), } s, err := b.StartSync(p.currentPosition) if err != nil { log.Infof("Start sync error: %v\n", err) return fmt.Errorf("Start sync: %w", err) } var finishError error FOR: for { e, err := s.GetEvent(p.ctx) if err != nil { if err == context.DeadlineExceeded { log.Warnf("Waiting for timeout(10s), no new event is generated, automatically stop: %v\n", err) // err = fmt.Errorf("Waiting for timeout(10s), no new event is generated, automatically stop: %v\n", err) err = nil } else { log.Errorf("Get event error: %v\n", err) } finishError = fmt.Errorf("get binlog event: %w", err) break FOR } select { case <-p.ctx.Done(): finishError = context.Canceled break FOR default: ok, err := p.parseSingleEvent(e) if err != nil { finishError = err break FOR } if !ok { break FOR } } } p.running = false return finishError } // checkFinish 检查是否需要结束循环 func (p *BinlogParserStats) checkFinish(currentPosition *mysql.Position) int { returnValue := -1 // 满足以下任一条件时结束循环 // * binlog文件超出指定结束文件 // * 超出指定结束文件的指定位置 // * 超出当前最新binlog位置 // 当前binlog解析位置 相对于配置的结束位置/最新位置 // 超出时返回 1 // 等于时返回 0 // 小于时返回 -1 // 根据是否为事件开始做不同判断 // 事件开始时做大于判断 // 事件结束时做等于判断 var stopMsg string if p.stopFile != "" && currentPosition.Name > p.stopFile { stopMsg = "超出指定结束文件" returnValue = 1 } else if p.cfg.StopPosition > 0 && currentPosition.Name == p.stopFile { if currentPosition.Pos > uint32(p.cfg.StopPosition) { stopMsg = "超出指定位置" returnValue = 1 } else if currentPosition.Pos == uint32(p.cfg.StopPosition) { stopMsg = "超出指定位置" returnValue = 0 } } if p.masterStatus != nil { if currentPosition.Name > p.masterStatus.File || currentPosition.Name == p.masterStatus.File && currentPosition.Pos > uint32(p.masterStatus.Position) { stopMsg = "超出最新binlog位置" returnValue = 1 } else if currentPosition.Name == p.masterStatus.File && currentPosition.Pos == uint32(p.masterStatus.Position) { stopMsg = "超出最新binlog位置" returnValue = 0 } } if stopMsg != "" { log.WithFields(log.Fields{ "当前文件": currentPosition.Name, "结束文件": p.stopFile, "当前位置": currentPosition.Pos, "结束位置": p.cfg.StopPosition, }).Info(stopMsg) } return returnValue } // clear 压缩完成或者没有数据时删除文件 func (p *BinlogParserStats) clear() { err := os.Remove(p.outputFileName) if err != nil { log.Error(err) log.Errorf("删除文件失败! %s", p.outputFileName) } } func (p *BinlogParserStats) Stop() { log.Warn("Process killed") if p.cancelFn != nil { p.cancelFn() } p.running = false } func (p *BinlogParserStats) writeString(str string) { if len(p.outputFileName) > 0 { p.bufferWriter.WriteString(str) } else { fmt.Print(str) } } // NewBinlogParserStats binlog解析器 func NewBinlogParserStats(cfg *BinlogParserConfig) (*BinlogParserStats, error) { p := new(BinlogParserStats) p.allTables = make(map[uint64]*Table) cfg.beginTime = time.Now().Unix() p.cfg = cfg p.OnlyDatabases = make(map[string]bool) // [table_name] = db_name p.OnlyTables = make(map[string]string) p.SqlTypes = make(map[string]bool) p.startFile = cfg.StartFile p.stopFile = cfg.StopFile if cfg.OutputFileStr != "" { p.outputFileName = cfg.OutputFileStr } else { p.outputFile = os.Stdout } // 本地解析模式,host为空,表名为SQL文件 if p.cfg.Host == "" { p.localMode = true } if !p.localMode { var err error p.db, err = p.getDB() if err != nil { return nil, err } defer p.db.Close() p.masterStatus, err = p.mysqlMasterStatus() if err != nil { return nil, err } if p.cfg.Debug { p.db.LogMode(true) } } if err := p.parserInit(); err != nil { return nil, err } p.ctx, p.cancelFn = context.WithCancel(context.Background()) p.stats = SummaryStats{} p.statsTables = make(map[string]*TableStats) return p, nil } // parserInit 解析服务初始化 func (p *BinlogParserStats) parserInit() error { defer timeTrack(time.Now(), "parserInit") if len(p.outputFileName) > 0 { var err error if Exists(p.outputFileName) { // p.checkError(errors.New("指定的文件已存在!")) // 不作追加,改为文件存在时清除内容,O_APPEND p.outputFile, err = os.OpenFile(p.outputFileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) } else { p.outputFile, err = os.Create(p.outputFileName) } p.checkError(err) // outputFile, err := os.OpenFile(outputFileStr, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) // if err != nil { // log.Info(index, "open file failed.", err.Error()) // break // } defer p.outputFile.Close() p.bufferWriter = bufio.NewWriter(p.outputFile) } if p.cfg.StartTime != "" { t, err := now.Parse(p.cfg.StartTime) if err != nil { return err } p.startTimestamp = uint32(t.Unix()) } if p.cfg.StopTime != "" { t, err := now.Parse(p.cfg.StopTime) if err != nil { return err } p.stopTimestamp = uint32(t.Unix()) } // 如果未指定开始文件,就自动解析 if len(p.startFile) == 0 { binlogs := p.autoParseBinlogPosition() if len(binlogs) == 0 { p.checkError(errors.New("无法获取master binlog")) } p.startFile = binlogs[0].Name if p.startTimestamp > 0 || p.stopTimestamp > 0 { for _, masterLog := range binlogs { timestamp, err := p.getBinlogFirstTimestamp(masterLog.Name) p.checkError(err) log.WithFields(log.Fields{ "起始时间": time.Unix(int64(timestamp), 0).Format(TimeFormat), "binlogFile": masterLog.Name, }).Info("binlog信息") if timestamp <= p.startTimestamp { p.startFile = masterLog.Name } if p.stopFile == "" && p.stopTimestamp > 0 && timestamp > p.stopTimestamp { p.stopFile = masterLog.Name } } } if len(p.startFile) == 0 { p.checkError(errors.New("未能解析指定时间段的binlog文件,请检查后重试!")) } log.Infof("根据指定的时间段,解析出的开始binlog文件是:%s,结束文件是:%s\n", p.startFile, p.stopFile) } // // 未指定结束文件时,仅解析该文件 // if len(p.stopFile) == 0 { // p.stopFile = p.startFile // } if len(p.cfg.SqlType) > 0 { for _, s := range strings.Split(p.cfg.SqlType, ",") { p.SqlTypes[s] = true } } else { p.SqlTypes["insert"] = true p.SqlTypes["update"] = true p.SqlTypes["delete"] = true } if len(p.cfg.Databases) > 0 { for _, db := range strings.Split(p.cfg.Databases, ",") { db = strings.ToLower(strings.Trim(db, " `")) p.OnlyDatabases[db] = true } } if len(p.cfg.Tables) > 0 { for _, s := range strings.Split(p.cfg.Tables, ",") { if strings.Contains(s, ".") { names := strings.SplitN(s, ".", 2) db, table := names[0], names[1] db = strings.ToLower(strings.Trim(db, " `")) table = strings.ToLower(strings.Trim(table, " `")) p.OnlyTables[table] = db } else { key := strings.ToLower(strings.Trim(s, " `")) p.OnlyTables[key] = "" } } } // log.Infof("dbs: %#v", p.OnlyDatabases) // log.Infof("tbls: %#v", p.OnlyTables) return nil } // getBinlogFirstTimestamp 获取binlog文件第一个时间戳 func (p *BinlogParserStats) getBinlogFirstTimestamp(file string) (uint32, error) { logLevel := log.GetLevel() defer func() { log.SetLevel(logLevel) }() // 临时调整日志级别,忽略close sync异常 log.SetLevel(log.FatalLevel) cfg := replication.BinlogSyncerConfig{ ServerID: 2000000110, Flavor: p.cfg.Flavor, Host: p.cfg.Host, Port: p.cfg.Port, User: p.cfg.User, Password: p.cfg.Password, // RawModeEnabled: p.cfg.RawMode, // SemiSyncEnabled: p.cfg.SemiSync, } b := replication.NewBinlogSyncer(cfg) pos := mysql.Position{Name: file, Pos: uint32(4)} s, err := b.StartSync(pos) if err != nil { return 0, err } defer func() { b.Close() }() for { // return e, err := s.GetEvent(context.Background()) if err != nil { // b.Close() return 0, err } // log.Infof("事件类型:%s", e.Header.EventType) if e.Header.Timestamp > 0 { // b.Close() return e.Header.Timestamp, nil } } } func (p *BinlogParserStats) checkError(e error) { if e != nil { log.Error(e) panic(e) } } func (p *BinlogParserStats) mysqlMasterStatus() (*MasterStatus, error) { defer timeTrack(time.Now(), "mysqlMasterStatus") r := MasterStatus{} p.db.Raw("SHOW MASTER STATUS").Scan(&r) return &r, nil } func (p *BinlogParserStats) autoParseBinlogPosition() []MasterLog { if p.binlogs != nil { return p.binlogs } var binlogIndex []MasterLog p.db.Raw("SHOW MASTER LOGS").Scan(&binlogIndex) p.binlogs = binlogIndex return binlogIndex } func getDBTableKey(db, table string) string { return fmt.Sprintf("`%s`.`%s`", strings.ToLower(db), strings.ToLower(table)) } func (p *BinlogParserStats) parseSingleEvent(e *replication.BinlogEvent) (ok bool, err error) { // 是否继续,默认为true ok = true finishFlag := -1 if e.Header.LogPos > 0 { p.currentPosition.Pos = e.Header.LogPos } if e.Header.EventType == replication.ROTATE_EVENT { if event, ok := e.Event.(*replication.RotateEvent); ok { p.currentPosition = mysql.Position{ Name: string(event.NextLogName), Pos: uint32(event.Position)} } } if e.Header.Timestamp > 0 { if p.startTimestamp > 0 && e.Header.Timestamp < p.startTimestamp { return } if p.stats.StartTime.IsZero() { p.stats.StartTime = time.Unix(int64(e.Header.Timestamp), 0) } p.stats.EndTime = time.Unix(int64(e.Header.Timestamp), 0) if p.stopTimestamp > 0 && e.Header.Timestamp > p.stopTimestamp { log.Warn("已超出结束时间") return false, nil } } p.currentTimtstamp = e.Header.Timestamp finishFlag = p.checkFinish(&p.currentPosition) if finishFlag == 1 { log.Warn("is finish") return false, nil } switch event := e.Event.(type) { case *replication.RowsEvent: key := getDBTableKey(string(event.Table.Schema), string(event.Table.Table)) if _, ok := p.statsTables[key]; !ok { p.statsTables[key] = &TableStats{ DB: string(event.Table.Schema), Table: string(event.Table.Table), } } currentTable := p.statsTables[key] switch e.Header.EventType { case replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2: currentTable.Inserts += len(event.Rows) currentTable.Total += len(event.Rows) case replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2: currentTable.Deletes += len(event.Rows) currentTable.Total += len(event.Rows) case replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2: currentTable.Updates += len(event.Rows) / 2 currentTable.Total += len(event.Rows) } } // 再次判断是否到结束位置,以免处在临界值,且无新日志时程序卡住 if e.Header.Timestamp > 0 { if p.stopTimestamp > 0 && e.Header.Timestamp > p.stopTimestamp { log.Warn("已超出结束时间") return false, nil } } if !p.running { log.Warn("进程手动中止") return false, nil } if finishFlag > -1 { return false, nil } return } func (p *BinlogParserStats) parserFile() error { defer timeTrack(time.Now(), "parserFile") defer func() { if p.outputFile != nil { p.outputFile.Close() } }() var err error p.running = true if len(p.outputFileName) > 0 { if Exists(p.outputFileName) { // p.checkError(errors.New("指定的文件已存在!")) // 不作追加,改为文件存在时清除内容,O_APPEND p.outputFile, err = os.OpenFile(p.outputFileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666) } else { p.outputFile, err = os.Create(p.outputFileName) } if err != nil { return err } p.bufferWriter = bufio.NewWriter(p.outputFile) } b := replication.NewBinlogParser() b.SetUseDecimal(true) p.currentPosition = mysql.Position{ Name: p.startFile, Pos: uint32(p.cfg.StartPosition), } f := func(e *replication.BinlogEvent) error { ok, err := p.parseSingleEvent(e) if err != nil { return err } if !ok { b.Stop() } // 再次判断是否到结束位置,以免处在临界值,且无新日志时程序卡住 if e.Header.Timestamp > 0 { if p.stopTimestamp > 0 && e.Header.Timestamp >= p.stopTimestamp { log.Warn("已超出结束时间") b.Stop() return nil } } return nil } err = b.ParseFile(p.startFile, int64(p.cfg.StartPosition), f) if err != nil { println(err.Error()) } p.stats.Tables = make([]TableStats, 0, len(p.statsTables)) for _, t := range p.statsTables { p.stats.Inserts += t.Inserts p.stats.Updates += t.Updates p.stats.Deletes += t.Deletes p.stats.Total += t.Total p.stats.Tables = append(p.stats.Tables, *t) } p.writeString(p.stats.String()) return nil } ================================================ FILE: core/parser_test.go ================================================ package core_test import ( "bufio" "context" "flag" "fmt" "io" "io/ioutil" "os" "strings" "testing" "time" "github.com/go-mysql-org/go-mysql/client" "github.com/go-mysql-org/go-mysql/mysql" _ "github.com/go-sql-driver/mysql" "github.com/hanchuanchuan/bingo2sql/core" . "github.com/pingcap/check" log "github.com/sirupsen/logrus" ) // Use docker mysql to test, mysql is 3306, mariadb is 3316 var testHost = flag.String("host", "127.0.0.1", "MySQL master host") var testOutputLogs = flag.Bool("out", false, "output binlog event") var allTables = map[string]string{ "test_json_v2": `CREATE TABLE IF NOT EXISTS test_json_v2 ( id INT, c JSON, PRIMARY KEY (id) ) ENGINE=InnoDB`, "test_replication": `CREATE TABLE IF NOT EXISTS test_replication ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, str VARCHAR(256), f FLOAT, d DOUBLE, de DECIMAL(10,2), i INT, bi BIGINT, e enum ("e1", "e2"), b BIT(8), y YEAR, da DATE, ts TIMESTAMP, dt DATETIME, tm TIME, t TEXT, bb BLOB, se SET('a', 'b', 'c'), PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8`, "test_json": `CREATE TABLE IF NOT EXISTS test_json ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, c1 JSON, c2 DECIMAL(10, 0), PRIMARY KEY (id) ) ENGINE=InnoDB`, "test_geo": `CREATE TABLE IF NOT EXISTS test_geo (id int auto_increment primary key, g GEOMETRY)`, "test_parse_time": `CREATE TABLE IF NOT EXISTS test_parse_time ( id int auto_increment primary key, a1 DATETIME, a2 DATETIME(3), a3 DATETIME(6), b1 TIMESTAMP, b2 TIMESTAMP(3) , b3 TIMESTAMP(6))`, "test_simple": `CREATE TABLE IF NOT EXISTS test_simple ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, c1 varchar(100), c2 int, PRIMARY KEY (id) ) ENGINE=InnoDB`, "test_long_text": `CREATE TABLE IF NOT EXISTS test_long_text ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, c1 longtext, PRIMARY KEY (id) ) ENGINE=InnoDB`, "test_generated": `CREATE TABLE IF NOT EXISTS test_generated ( id int primary key, price int, number int, total int generated always as (price*number))`, } var ( // 解析binlog生成的SQL文件 binlogOutputFile string = "binlog_output.sql" // 本地方式解析binlog生成的SQL文件 localOutputFile string = "binlog_output_local.sql" // 表结构文件. 用于本地解析 tableSchemaFile string = "table_schema.sql" ) var ( defaultConfig core.BinlogParserConfig localConfig core.BinlogParserConfig ) var _ = Suite(&testParserSuite{}) func TestBinLogSyncer(t *testing.T) { TestingT(t) } type testParserSuite struct { c *client.Conn flavor string config core.BinlogParserConfig // 远程解析 localConfig core.BinlogParserConfig // 本地解析 } func (t *testParserSuite) SetUpSuite(c *C) { defaultConfig = core.BinlogParserConfig{ Host: *testHost, Port: 3306, User: "test", Password: "test", StartFile: "mysql-bin.000001", Databases: "test", OutputFileStr: binlogOutputFile, } localConfig = core.BinlogParserConfig{ // 通过setBinlogDir自动获取 // StartFile: "mysql-bin.000001", Databases: "test", Tables: tableSchemaFile, OutputFileStr: localOutputFile, } t.config = defaultConfig t.localConfig = localConfig t.initTableSchema() t.setBinlogDir(c) t.createTables(c) log.SetLevel(log.InfoLevel) } func (t *testParserSuite) TearDownSuite(c *C) { os.Remove(binlogOutputFile) os.Remove(localOutputFile) os.Remove(tableSchemaFile) } func (t *testParserSuite) SetUpTest(c *C) { } func (t *testParserSuite) TearDownTest(c *C) { t.reset() // if t.b != nil { // t.b.Close() // t.b = nil // } if t.c != nil { t.c.Close() t.c = nil } } func (t *testParserSuite) testExecute(c *C, query ...string) { for _, q := range query { _, err := t.c.Execute(q) c.Assert(err, IsNil) } } func (t *testParserSuite) setBinlogDir(c *C) { if t.c == nil { t.setupTest(c, mysql.MySQLFlavor) } result, err := t.c.Execute("show variables like 'log_bin_basename'") c.Assert(err, IsNil) basename, err := result.GetString(0, 1) c.Assert(err, IsNil) basename = strings.Replace(basename, "/data/mysql", "/Users/hanchuanchuan", 1) t.localConfig.StartFile = fmt.Sprintf("%s.000001", basename) localConfig.StartFile = fmt.Sprintf("%s.000001", basename) log.Infof("开始本地日志文件:%s", localConfig.StartFile) result, err = t.c.Execute("show master logs") c.Assert(err, IsNil) basename, err = result.GetString(0, 0) c.Assert(err, IsNil) t.config.StartFile = basename defaultConfig.StartFile = basename log.Infof("开始日志文件:%s", basename) } func (t *testParserSuite) SetFlashback(v bool) { t.config.Flashback = v t.localConfig.Flashback = v } func (t *testParserSuite) SetMinimalUpdate(v bool) { t.config.MinimalUpdate = v t.localConfig.MinimalUpdate = v } func (t *testParserSuite) SetRemovePrimary(v bool) { t.config.RemovePrimary = v t.localConfig.RemovePrimary = v } func (t *testParserSuite) SetSQLType(v string) { t.config.SqlType = v t.localConfig.SqlType = v } func (t *testParserSuite) SetIncludeGtids(v string) { t.config.IncludeGtids = v t.localConfig.IncludeGtids = v } func (t *testParserSuite) setupTest(c *C, flavor string) { var port uint16 = 3306 switch flavor { case mysql.MariaDBFlavor: port = 3316 } t.flavor = flavor var err error if t.c != nil { t.c.Close() } // db, err := sql.Open(sqlType, "user:password@tcp(127.0.0.1:3306)/database?multiStatements=true") // if err != nil { // log.Fatalf("open mysql err: %s", err) // } t.c, err = client.Connect(fmt.Sprintf("%s:%d", *testHost, port), "test", "test", "") c.Assert(err, IsNil) // _, err = t.c.Execute("CREATE DATABASE IF NOT EXISTS test") // c.Assert(err, IsNil) _, err = t.c.Execute("USE test") c.Assert(err, IsNil) _, err = t.c.Execute("set binlog_format = 'row'") c.Assert(err, IsNil) } func (t *testParserSuite) getThreadID(c *C) uint32 { result, err := t.c.Execute("select connection_id()") c.Assert(err, IsNil) threadID, err := result.GetInt(0, 0) c.Assert(err, IsNil) // log.Errorf("%#v", threadID) return uint32(threadID) } func (t *testParserSuite) getServerUUID(c *C) string { result, err := t.c.Execute("show variables like 'server_uuid'") c.Assert(err, IsNil) uuid, err := result.GetString(0, 1) c.Assert(err, IsNil) log.Infof("server uuid: %#v", uuid) result, err = t.c.Execute("show master status;") c.Assert(err, IsNil) file, _ := result.GetString(0, 0) gtidSet, _ := result.GetString(0, 4) log.Infof("binlog: %s, gtid set: %s", file, gtidSet) return uuid } // func (t *testParserSuite) testPositionSync(c *C) { // //get current master binlog file and position // r, err := t.c.Execute("SHOW MASTER STATUS") // c.Assert(err, IsNil) // binFile, _ := r.GetString(0, 0) // binPos, _ := r.GetInt(0, 1) // s, err := t.b.StartSync(mysql.Position{Name: binFile, Pos: uint32(binPos)}) // c.Assert(err, IsNil) // // Test re-sync. // time.Sleep(100 * time.Millisecond) // t.b.c.SetReadDeadline(time.Now().Add(time.Millisecond)) // time.Sleep(100 * time.Millisecond) // t.testSync(c, s) // } // func (t *testParserSuite) TestMysqlPositionSync(c *C) { // t.setupTest(c, mysql.MySQLFlavor) // t.testPositionSync(c) // } // func (t *testParserSuite) TestMysqlGTIDSync(c *C) { // t.setupTest(c, mysql.MySQLFlavor) // r, err := t.c.Execute("SELECT @@gtid_mode") // c.Assert(err, IsNil) // modeOn, _ := r.GetString(0, 0) // if modeOn != "ON" { // c.Skip("GTID mode is not ON") // } // r, err = t.c.Execute("SHOW GLOBAL VARIABLES LIKE 'SERVER_UUID'") // c.Assert(err, IsNil) // var masterUuid uuid.UUID // if s, _ := r.GetString(0, 1); len(s) > 0 && s != "NONE" { // masterUuid, err = uuid.FromString(s) // c.Assert(err, IsNil) // } // set, _ := mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d-%d", masterUuid.String(), 1, 2)) // s, err := t.b.StartSyncGTID(set) // c.Assert(err, IsNil) // t.testSync(c, s) // } // func (t *testParserSuite) TestMariadbPositionSync(c *C) { // t.setupTest(c, mysql.MariaDBFlavor) // t.testPositionSync(c) // } // func (t *testParserSuite) TestMariadbGTIDSync(c *C) { // t.setupTest(c, mysql.MariaDBFlavor) // // get current master gtid binlog pos // r, err := t.c.Execute("SELECT @@gtid_binlog_pos") // c.Assert(err, IsNil) // str, _ := r.GetString(0, 0) // set, _ := mysql.ParseMariadbGTIDSet(str) // s, err := t.b.StartSyncGTID(set) // c.Assert(err, IsNil) // t.testSync(c, s) // } // func (t *testParserSuite) TestMariadbAnnotateRows(c *C) { // t.setupTest(c, mysql.MariaDBFlavor) // t.b.cfg.DumpCommandFlag = BINLOG_SEND_ANNOTATE_ROWS_EVENT // t.testPositionSync(c) // } // func (t *testParserSuite) TestMysqlSemiPositionSync(c *C) { // t.setupTest(c, mysql.MySQLFlavor) // t.b.cfg.SemiSyncEnabled = true // t.testPositionSync(c) // } // func (t *testParserSuite) TestMysqlBinlogCodec(c *C) { // t.setupTest(c, mysql.MySQLFlavor) // t.testExecute(c, "RESET MASTER") // var wg sync.WaitGroup // wg.Add(1) // defer wg.Wait() // go func() { // defer wg.Done() // t.testSync(c, nil) // t.testExecute(c, "FLUSH LOGS") // t.testSync(c, nil) // }() // binlogDir := "./var" // os.RemoveAll(binlogDir) // err := t.b.StartBackup(binlogDir, mysql.Position{Name: "", Pos: uint32(0)}, 2*time.Second) // c.Assert(err, IsNil) // p := NewBinlogParser() // p.SetVerifyChecksum(true) // f := func(e *BinlogEvent) error { // if *testOutputLogs { // e.Dump(os.Stdout) // os.Stdout.Sync() // } // return nil // } // dir, err := os.Open(binlogDir) // c.Assert(err, IsNil) // defer dir.Close() // files, err := dir.Readdirnames(-1) // c.Assert(err, IsNil) // for _, file := range files { // err = p.ParseFile(path.Join(binlogDir, file), 0, f) // c.Assert(err, IsNil) // } // } func (t *testParserSuite) checkBinlog(c *C, sqls ...string) { binlogs := t.getBinlog(c) c.Assert(len(binlogs), Equals, len(sqls), Commentf("%#v", binlogs)) for i, line := range binlogs { c.Assert(line, Equals, sqls[i], Commentf("%#v", binlogs)) } } func (t *testParserSuite) getBinlog(c *C) []string { // 在线方式解析 resultOnline := t.getBinlogWithConfig(c, &t.config) // 判断本地文件是否存在 if _, err := os.Stat(t.localConfig.StartFile); err == nil { // if runtime.GOOS == "linux" { // 本地解析 resultLocal := t.getBinlogWithConfig(c, &t.localConfig) c.Assert(len(resultOnline), Equals, len(resultLocal), Commentf("local: %#v, online: %#v", resultLocal, resultOnline)) for i, line := range resultOnline { c.Assert(line, Equals, resultLocal[i], Commentf("local: %#v, online: %#v", resultLocal[i], resultOnline)) } } else { log.Warnf("跳过本地文件解析! 本地文件不存在:%s", t.localConfig.StartFile) } return resultOnline } // getBinlogWithConfig 根据配置文件 func (t *testParserSuite) getBinlogWithConfig(c *C, config *core.BinlogParserConfig) []string { p, err := core.NewBinlogParser(context.Background(), config) c.Assert(err, IsNil) err = p.Parser() c.Assert(err, IsNil) fileObj, err := os.Open(config.OutputFileStr) c.Assert(err, IsNil) defer fileObj.Close() //一个文件对象本身是实现了io.Reader的 使用bufio.NewReader去初始化一个Reader对象，存在buffer中的，读取一次就会被清空 reader := bufio.NewReader(fileObj) var buf []string for { line, err := reader.ReadString('\n') if err != nil { if err != io.EOF { c.Assert(err, IsNil) return nil } break } line = strings.TrimSpace(line) if strings.HasPrefix(line, "# ") || line == "" { continue } if strings.Contains(line, "# ") { line = line[0:strings.Index(line, "# ")] } buf = append(buf, strings.TrimSpace(line)) } return buf } func (t *testParserSuite) reset() { t.config = defaultConfig t.localConfig = localConfig // log.SetLevel(log.ErrorLevel) } func (t *testParserSuite) TestSync(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, `RESET MASTER;`, // "SET SESSION binlog_format = 'MIXED'", `DROP TABLE IF EXISTS test_replication`, `CREATE TABLE test_replication ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, str VARCHAR(256), f FLOAT, d DOUBLE, de DECIMAL(10,2), i INT, bi BIGINT, e enum ("e1", "e2"), b BIT(8), y YEAR, da DATE, ts TIMESTAMP, dt DATETIME, tm TIME, t TEXT, bb BLOB, se SET('a', 'b', 'c'), PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8`) //use row format t.testExecute(c, `INSERT INTO test_replication (str, f, i, e, b, y, da, ts, dt, tm, de, t, bb, se) VALUES ("3", -3.14, 10, "e1", 0b0011, 1985, "2012-05-07", "2012-05-07 14:01:01", "2012-05-07 14:01:01", "14:01:01", -45363.64, "abc", "12345", "a,b")`) t.checkBinlog(c, "INSERT INTO `test`.`test_replication`(`id`,`str`,`f`,`d`,`de`,`i`,`bi`,`e`,`b`,`y`,`da`,`ts`,`dt`,`tm`,`t`,`bb`,`se`) VALUES(1,'3',-3.14,NULL,-45363.64,10,NULL,1,3,1985,'2012-05-07','2012-05-07 14:01:01','2012-05-07 14:01:01','14:01:01','abc','12345',3);") t.checkBinlog(c, "INSERT INTO `test`.`test_replication`(`id`,`str`,`f`,`d`,`de`,`i`,`bi`,`e`,`b`,`y`,`da`,`ts`,`dt`,`tm`,`t`,`bb`,`se`) VALUES(1,'3',-3.14,NULL,-45363.64,10,NULL,1,3,1985,'2012-05-07','2012-05-07 14:01:01','2012-05-07 14:01:01','14:01:01','abc','12345',3);") t.SetFlashback(true) t.checkBinlog(c, "DELETE FROM `test`.`test_replication` WHERE `id`=1;") t.SetFlashback(false) t.SetRemovePrimary(true) t.checkBinlog(c, "INSERT INTO `test`.`test_replication`(`str`,`f`,`d`,`de`,`i`,`bi`,`e`,`b`,`y`,`da`,`ts`,`dt`,`tm`,`t`,`bb`,`se`) VALUES('3',-3.14,NULL,-45363.64,10,NULL,1,3,1985,'2012-05-07','2012-05-07 14:01:01','2012-05-07 14:01:01','14:01:01','abc','12345',3);") } func (t *testParserSuite) TestParseDDL(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, `RESET MASTER;`, `DROP TABLE IF EXISTS test_replication`, `CREATE TABLE test_replication ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, str VARCHAR(256), f FLOAT, d DOUBLE, de DECIMAL(10,2), i INT, bi BIGINT, e enum ("e1", "e2"), b BIT(8), y YEAR, da DATE, ts TIMESTAMP, dt DATETIME, tm TIME, t TEXT, bb BLOB, se SET('a', 'b', 'c'), PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8`) //use row format t.testExecute(c, `INSERT INTO test_replication (str, f, i, e, b, y, da, ts, dt, tm, de, t, bb, se) VALUES ("3", -3.14, 10, "e1", 0b0011, 1985, "2012-05-07", "2012-05-07 14:01:01", "2012-05-07 14:01:01", "14:01:01", -45363.64, "abc", "12345", "a,b")`) t.config.ParseDDL = true t.localConfig.ParseDDL = true t.checkBinlog(c, "USE `test`;", "DROP TABLE IF EXISTS `test_replication` /* generated by server */;", "USE `test`;", "CREATE TABLE test_replication (", "id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT,", "str VARCHAR(256),", "f FLOAT,", "d DOUBLE,", "de DECIMAL(10,2),", "i INT,", "bi BIGINT,", "e enum (\"e1\", \"e2\"),", "b BIT(8),", "y YEAR,", "da DATE,", "ts TIMESTAMP,", "dt DATETIME,", "tm TIME,", "t TEXT,", "bb BLOB,", "se SET('a', 'b', 'c'),", "PRIMARY KEY (id)", ") ENGINE=InnoDB DEFAULT CHARSET=utf8;", "INSERT INTO `test`.`test_replication`(`id`,`str`,`f`,`d`,`de`,`i`,`bi`,`e`,`b`,`y`,`da`,`ts`,`dt`,`tm`,`t`,`bb`,`se`) VALUES(1,'3',-3.14,NULL,-45363.64,10,NULL,1,3,1985,'2012-05-07','2012-05-07 14:01:01','2012-05-07 14:01:01','14:01:01','abc','12345',3);", ) t.SetFlashback(true) t.checkBinlog(c, "DELETE FROM `test`.`test_replication` WHERE `id`=1;", ) } func (t *testParserSuite) TestStopTime(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, `RESET MASTER;`, `DROP TABLE IF EXISTS test_replication`, `CREATE TABLE test_replication ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, str VARCHAR(256), f FLOAT, d DOUBLE, de DECIMAL(10,2), i INT, bi BIGINT, e enum ("e1", "e2"), b BIT(8), y YEAR, da DATE, ts TIMESTAMP, dt DATETIME, tm TIME, t TEXT, bb BLOB, se SET('a', 'b', 'c'), PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8`) //use row format t.testExecute(c, `INSERT INTO test_replication (str, f, i, e, b, y, da, ts, dt, tm, de, t, bb, se) VALUES ("3", -3.14, 10, "e1", 0b0011, 1985, "2012-05-07", "2012-05-07 14:01:01", "2012-05-07 14:01:01", "14:01:01", -45363.64, "abc", "12345", "a,b")`) t.config.StartTime = time.Now().Add(-10 * time.Minute).Format("2006-01-02 15:04") t.localConfig.StartTime = t.config.StartTime t.checkBinlog(c, "INSERT INTO `test`.`test_replication`(`id`,`str`,`f`,`d`,`de`,`i`,`bi`,`e`,`b`,`y`,`da`,`ts`,`dt`,`tm`,`t`,`bb`,`se`) VALUES(1,'3',-3.14,NULL,-45363.64,10,NULL,1,3,1985,'2012-05-07','2012-05-07 14:01:01','2012-05-07 14:01:01','14:01:01','abc','12345',3);") t.config.StopTime = time.Now().Add(-5 * time.Minute).Format("2006-01-02 15:04") t.localConfig.StopTime = t.config.StopTime t.SetFlashback(true) // 时间限制范围内无数据 t.checkBinlog(c) t.config.StopTime = time.Now().Add(time.Minute).Format("2006-01-02 15:04:05") t.localConfig.StopTime = t.config.StopTime t.checkBinlog(c, "DELETE FROM `test`.`test_replication` WHERE `id`=1;") t.SetFlashback(false) t.SetRemovePrimary(true) t.checkBinlog(c, "INSERT INTO `test`.`test_replication`(`str`,`f`,`d`,`de`,`i`,`bi`,`e`,`b`,`y`,`da`,`ts`,`dt`,`tm`,`t`,`bb`,`se`) VALUES('3',-3.14,NULL,-45363.64,10,NULL,1,3,1985,'2012-05-07','2012-05-07 14:01:01','2012-05-07 14:01:01','14:01:01','abc','12345',3);") } func (t *testParserSuite) TestGeometry(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, "RESET MASTER", "DROP TABLE IF EXISTS test_geo", `CREATE TABLE test_geo (id int auto_increment primary key, g GEOMETRY)`, ) tbls := []string{ `INSERT INTO test_geo(g) VALUES (POINT(1, 1))`, `INSERT INTO test_geo(g) VALUES (LINESTRING(POINT(0,0), POINT(1,1), POINT(2,2)))`, `DELETE from test_geo where id>0`, } t.testExecute(c, tbls...) t.SetFlashback(true) t.checkBinlog(c, "DELETE FROM `test`.`test_geo` WHERE `id`=1;", "DELETE FROM `test`.`test_geo` WHERE `id`=2;", "INSERT INTO `test`.`test_geo`(`id`,`g`) VALUES(1,'\\0\\0\\0\\0\x01\x01\\0\\0\\0\\0\\0\\0\\0\\0\\0\xf0?\\0\\0\\0\\0\\0\\0\xf0?');", "INSERT INTO `test`.`test_geo`(`id`,`g`) VALUES(2,'\\0\\0\\0\\0\x01\x02\\0\\0\\0\x03\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\\0\xf0?\\0\\0\\0\\0\\0\\0\xf0?\\0\\0\\0\\0\\0\\0\\0@\\0\\0\\0\\0\\0\\0\\0@');", ) } func (t *testParserSuite) TestDatetime(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, "RESET MASTER", `SET sql_mode=''`, `DROP TABLE IF EXISTS test_parse_time`, `CREATE TABLE test_parse_time ( id int auto_increment primary key, a1 DATETIME, a2 DATETIME(3), a3 DATETIME(6), b1 TIMESTAMP, b2 TIMESTAMP(3) , b3 TIMESTAMP(6))`, ) t.testExecute(c, `INSERT INTO test_parse_time(a1,a2,a3,b1,b2,b3) VALUES ("2014-09-08 17:51:04.123456", "2014-09-08 17:51:04.123456", "2014-09-08 17:51:04.123456", "2014-09-08 17:51:04.123456","2014-09-08 17:51:04.123456","2014-09-08 17:51:04.123456"), ("0000-00-00 00:00:00.000000", "0000-00-00 00:00:00.000000", "0000-00-00 00:00:00.000000", "0000-00-00 00:00:00.000000", "0000-00-00 00:00:00.000000", "0000-00-00 00:00:00.000000"), ("2014-09-08 17:51:04.000456", "2014-09-08 17:51:04.000456", "2014-09-08 17:51:04.000456", "2014-09-08 17:51:04.000456","2014-09-08 17:51:04.000456","2014-09-08 17:51:04.000456")`, `delete from test_parse_time where id > 0`) t.SetFlashback(true) t.checkBinlog(c, "DELETE FROM `test`.`test_parse_time` WHERE `id`=1;", "DELETE FROM `test`.`test_parse_time` WHERE `id`=2;", "DELETE FROM `test`.`test_parse_time` WHERE `id`=3;", "INSERT INTO `test`.`test_parse_time`(`id`,`a1`,`a2`,`a3`,`b1`,`b2`,`b3`) VALUES(1,'2014-09-08 17:51:04','2014-09-08 17:51:04.123','2014-09-08 17:51:04.123456','2014-09-08 17:51:04','2014-09-08 17:51:04.123','2014-09-08 17:51:04.123456');", "INSERT INTO `test`.`test_parse_time`(`id`,`a1`,`a2`,`a3`,`b1`,`b2`,`b3`) VALUES(2,'0000-00-00 00:00:00','0000-00-00 00:00:00.000','0000-00-00 00:00:00.000000','0000-00-00 00:00:00','0000-00-00 00:00:00.000','0000-00-00 00:00:00.000000');", "INSERT INTO `test`.`test_parse_time`(`id`,`a1`,`a2`,`a3`,`b1`,`b2`,`b3`) VALUES(3,'2014-09-08 17:51:04','2014-09-08 17:51:04.000','2014-09-08 17:51:04.000456','2014-09-08 17:51:04','2014-09-08 17:51:04.000','2014-09-08 17:51:04.000456');", ) } func (t *testParserSuite) TestBinlogRowImageMinimal(c *C) { t.setupTest(c, mysql.MySQLFlavor) id := 100 t.testExecute(c, `RESET MASTER;`, "SET SESSION binlog_row_image = 'MINIMAL'", fmt.Sprintf(`INSERT INTO test_replication (id, str, f, i, bb, de) VALUES (%d, "4", -3.14, 100, "abc", -45635.64)`, id), fmt.Sprintf(`UPDATE test_replication SET f = -12.14, de = 555.34 WHERE id = %d`, id), fmt.Sprintf(`DELETE FROM test_replication WHERE id = %d`, id)) t.SetSQLType("update") t.checkBinlog(c, "UPDATE `test`.`test_replication` SET `id`=NULL, `str`=NULL, `f`=-12.14, `d`=NULL, `de`=555.34, `i`=NULL, `bi`=NULL, `e`=NULL, `b`=NULL, `y`=NULL, `da`=NULL, `ts`=NULL, `dt`=NULL, `tm`=NULL, `t`=NULL, `bb`=NULL, `se`=NULL WHERE `id`=100;") t.SetFlashback(true) t.SetMinimalUpdate(true) t.checkBinlog(c, "UPDATE `test`.`test_replication` SET `id`=100, `f`=NULL, `de`=NULL WHERE `id` IS NULL;", ) t.SetFlashback(false) t.checkBinlog(c, "UPDATE `test`.`test_replication` SET `id`=NULL, `f`=-12.14, `de`=555.34 WHERE `id`=100;", ) t.testExecute(c, "SET SESSION binlog_row_image = 'FULL'") } func (t *testParserSuite) TestMinimalUpdate(c *C) { t.setupTest(c, mysql.MySQLFlavor) id := 100 t.testExecute(c, `RESET MASTER;`, fmt.Sprintf(`INSERT INTO test_replication (id, str, f, i, bb, de) VALUES (%d, "4", -3.14, 100, "abc", -45635.64)`, id), fmt.Sprintf(`UPDATE test_replication SET f = -12.14, de = 555.34 WHERE id = %d`, id), fmt.Sprintf(`UPDATE test_replication SET str=null WHERE id = %d`, id), fmt.Sprintf(`DELETE FROM test_replication WHERE id = %d`, id)) t.SetSQLType("update") t.checkBinlog(c, "UPDATE `test`.`test_replication` SET `id`=100, `str`='4', `f`=-12.14, `d`=NULL, `de`=555.34, `i`=100, `bi`=NULL, `e`=NULL, `b`=NULL, `y`=NULL, `da`=NULL, `ts`=NULL, `dt`=NULL, `tm`=NULL, `t`=NULL, `bb`='abc', `se`=NULL WHERE `id`=100;", "UPDATE `test`.`test_replication` SET `id`=100, `str`=NULL, `f`=-12.14, `d`=NULL, `de`=555.34, `i`=100, `bi`=NULL, `e`=NULL, `b`=NULL, `y`=NULL, `da`=NULL, `ts`=NULL, `dt`=NULL, `tm`=NULL, `t`=NULL, `bb`='abc', `se`=NULL WHERE `id`=100;") t.SetFlashback(true) t.SetMinimalUpdate(true) t.checkBinlog(c, "UPDATE `test`.`test_replication` SET `f`=-3.14, `de`=-45635.64 WHERE `id`=100;", "UPDATE `test`.`test_replication` SET `str`='4' WHERE `id`=100;", ) t.SetFlashback(false) t.checkBinlog(c, "UPDATE `test`.`test_replication` SET `f`=-12.14, `de`=555.34 WHERE `id`=100;", "UPDATE `test`.`test_replication` SET `str`=NULL WHERE `id`=100;", ) } func (t *testParserSuite) TestFieldGenerated(c *C) { t.setupTest(c, mysql.MySQLFlavor) id := 1 t.testExecute(c, `RESET MASTER;`, `INSERT INTO test_generated(id,price,number) VALUES (1,1,1),(2,3,4)`, fmt.Sprintf(`UPDATE test_generated SET price = 10, number = 20 WHERE id = %d`, id), fmt.Sprintf(`delete from test_generated WHERE id = %d`, id), ) // t.SetSQLType("insert") t.SetSQLType("insert,update,delete") t.checkBinlog(c, "INSERT INTO `test`.`test_generated`(`id`,`price`,`number`) VALUES(1,1,1);", "INSERT INTO `test`.`test_generated`(`id`,`price`,`number`) VALUES(2,3,4);", "UPDATE `test`.`test_generated` SET `id`=1, `price`=10, `number`=20 WHERE `id`=1;", "DELETE FROM `test`.`test_generated` WHERE `id`=1;", ) t.SetFlashback(true) t.checkBinlog(c, "DELETE FROM `test`.`test_generated` WHERE `id`=1;", "DELETE FROM `test`.`test_generated` WHERE `id`=2;", "UPDATE `test`.`test_generated` SET `id`=1, `price`=1, `number`=1 WHERE `id`=1;", "INSERT INTO `test`.`test_generated`(`id`,`price`,`number`) VALUES(1,10,20);", ) } func (t *testParserSuite) TestTextMax(c *C) { t.setupTest(c, mysql.MySQLFlavor) id := 100 // 65536 = 64k value := strings.Repeat("a", 1024*1024*10) t.testExecute(c, `RESET MASTER;`, `DROP TABLE IF EXISTS test_long_text`, `CREATE TABLE test_long_text ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, c1 longtext, PRIMARY KEY (id) ) ENGINE=InnoDB`, fmt.Sprintf(`INSERT INTO test_long_text (id, c1) VALUES (%d, "")`, id), fmt.Sprintf(`UPDATE test_long_text SET c1 = '%s' WHERE id = %d`, value, id), fmt.Sprintf(`DELETE FROM test_long_text WHERE id = %d`, id)) t.checkBinlog(c, "INSERT INTO `test`.`test_long_text`(`id`,`c1`) VALUES(100,'');", fmt.Sprintf("UPDATE `test`.`test_long_text` SET `id`=100, `c1`='%s' WHERE `id`=100;", value), "DELETE FROM `test`.`test_long_text` WHERE `id`=100;", ) t.SetFlashback(true) t.SetMinimalUpdate(true) t.checkBinlog(c, "DELETE FROM `test`.`test_long_text` WHERE `id`=100;", "UPDATE `test`.`test_long_text` SET `c1`='' WHERE `id`=100;", fmt.Sprintf("INSERT INTO `test`.`test_long_text`(`id`,`c1`) VALUES(100,'%s');", value), ) } func (t *testParserSuite) TestUpdate2Null(c *C) { t.setupTest(c, mysql.MySQLFlavor) id := 100 t.testExecute(c, `RESET MASTER;`, fmt.Sprintf(`INSERT INTO test_replication (id,str, f, i, e, b, y, da, ts, dt, tm, de, t, bb, se) VALUES (%d,"3", -3.14, 10, "e1", 0b0011, 1985, "2012-05-07", "2012-05-07 14:01:01", "2012-05-07 14:01:01", "14:01:01", -45363.64, "abc", "12345", "a,b")`, id), fmt.Sprintf(`UPDATE test_replication SET str = null,f = null,d = null,de = null,i = null,bi = null,e = null,b = null,y = null,da = null,ts = null,dt = null,tm = null,t = null,bb = null WHERE id = %d`, id), fmt.Sprintf(`DELETE FROM test_replication WHERE id = %d`, id)) t.SetSQLType("update") t.checkBinlog(c, "UPDATE `test`.`test_replication` SET `id`=100, `str`=NULL, `f`=NULL, `d`=NULL, `de`=NULL, `i`=NULL, `bi`=NULL, `e`=NULL, `b`=NULL, `y`=NULL, `da`=NULL, `ts`=NULL, `dt`=NULL, `tm`=NULL, `t`=NULL, `bb`=NULL, `se`=3 WHERE `id`=100;") t.SetFlashback(true) t.SetMinimalUpdate(true) t.checkBinlog(c, "UPDATE `test`.`test_replication` SET `str`='3', `f`=-3.14, `de`=-45363.64, `i`=10, `e`=1, `b`=3, `y`=1985, `da`='2012-05-07', `ts`='2012-05-07 14:01:01', `dt`='2012-05-07 14:01:01', `tm`='14:01:01', `t`='abc', `bb`='12345' WHERE `id`=100;", ) t.SetFlashback(false) t.checkBinlog(c, "UPDATE `test`.`test_replication` SET `str`=NULL, `f`=NULL, `de`=NULL, `i`=NULL, `e`=NULL, `b`=NULL, `y`=NULL, `da`=NULL, `ts`=NULL, `dt`=NULL, `tm`=NULL, `t`=NULL, `bb`=NULL WHERE `id`=100;", ) } func (t *testParserSuite) TestRemovePrimary(c *C) { t.setupTest(c, mysql.MySQLFlavor) id := 100 t.testExecute(c, `RESET MASTER;`, fmt.Sprintf(`INSERT INTO test_replication (id, str, f, i, bb, de) VALUES (%d, "4", -3.14, 100, "abc", -45635.64)`, id), fmt.Sprintf(`DELETE FROM test_replication WHERE id = %d`, id)) t.checkBinlog(c, "INSERT INTO `test`.`test_replication`(`id`,`str`,`f`,`d`,`de`,`i`,`bi`,`e`,`b`,`y`,`da`,`ts`,`dt`,`tm`,`t`,`bb`,`se`) VALUES(100,'4',-3.14,NULL,-45635.64,100,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'abc',NULL);", "DELETE FROM `test`.`test_replication` WHERE `id`=100;") t.SetFlashback(true) t.checkBinlog(c, "DELETE FROM `test`.`test_replication` WHERE `id`=100;", "INSERT INTO `test`.`test_replication`(`id`,`str`,`f`,`d`,`de`,`i`,`bi`,`e`,`b`,`y`,`da`,`ts`,`dt`,`tm`,`t`,`bb`,`se`) VALUES(100,'4',-3.14,NULL,-45635.64,100,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'abc',NULL);", ) t.SetFlashback(false) t.SetRemovePrimary(true) t.checkBinlog(c, "INSERT INTO `test`.`test_replication`(`str`,`f`,`d`,`de`,`i`,`bi`,`e`,`b`,`y`,`da`,`ts`,`dt`,`tm`,`t`,`bb`,`se`) VALUES('4',-3.14,NULL,-45635.64,100,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'abc',NULL);", "DELETE FROM `test`.`test_replication` WHERE `id`=100;") } // TestThreadID 测试指定线程号,操作后断开重连 func (t *testParserSuite) TestThreadID(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, `RESET MASTER;`, `DROP TABLE IF EXISTS test_simple`, `DROP TABLE IF EXISTS test_simple`, `CREATE TABLE test_simple ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, c1 varchar(100), c2 int, PRIMARY KEY (id) ) ENGINE=InnoDB`) t.testExecute(c, `INSERT INTO test_simple (c1, c2) VALUES ('test',1)`, `DELETE FROM test_simple WHERE id > 0`) threadID := t.getThreadID(c) if t.c != nil { t.c.Close() t.c = nil } t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, `INSERT INTO test_simple (c1, c2) VALUES ('test2',2)`, `DELETE FROM test_simple WHERE id > 0`) // 未限制线程号时返回所有数据 t.checkBinlog(c, "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(1,'test',1);", "DELETE FROM `test`.`test_simple` WHERE `id`=1;", "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(2,'test2',2);", "DELETE FROM `test`.`test_simple` WHERE `id`=2;") t.config.ThreadID = threadID t.localConfig.ThreadID = t.config.ThreadID t.checkBinlog(c, "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(1,'test',1);", "DELETE FROM `test`.`test_simple` WHERE `id`=1;") t.SetFlashback(true) t.checkBinlog(c, "DELETE FROM `test`.`test_simple` WHERE `id`=1;", "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(1,'test',1);", ) } // TestInsert 测试insert func (t *testParserSuite) TestInsert(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, `RESET MASTER;`, `DROP TABLE IF EXISTS test_simple`, `CREATE TABLE test_simple ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, c1 varchar(100), c2 int, PRIMARY KEY (id) ) ENGINE=InnoDB`) t.testExecute(c, `INSERT INTO test_simple (c1, c2) VALUES ('a1',1),('a2',2),('a3',3),('a4',4),('a5',5)`, `DELETE FROM test_simple WHERE id > 0`) t.config.MinimalInsert = false t.localConfig.MinimalInsert = false t.checkBinlog(c, "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(1,'a1',1);", "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(2,'a2',2);", "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(3,'a3',3);", "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(4,'a4',4);", "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(5,'a5',5);", "DELETE FROM `test`.`test_simple` WHERE `id`=1;", "DELETE FROM `test`.`test_simple` WHERE `id`=2;", "DELETE FROM `test`.`test_simple` WHERE `id`=3;", "DELETE FROM `test`.`test_simple` WHERE `id`=4;", "DELETE FROM `test`.`test_simple` WHERE `id`=5;") t.config.MinimalInsert = true t.localConfig.MinimalInsert = true t.checkBinlog(c, "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(1,'a1',1),(2,'a2',2),(3,'a3',3),(4,'a4',4),(5,'a5',5);", "DELETE FROM `test`.`test_simple` WHERE `id`=1;", "DELETE FROM `test`.`test_simple` WHERE `id`=2;", "DELETE FROM `test`.`test_simple` WHERE `id`=3;", "DELETE FROM `test`.`test_simple` WHERE `id`=4;", "DELETE FROM `test`.`test_simple` WHERE `id`=5;") t.SetFlashback(true) t.checkBinlog(c, "DELETE FROM `test`.`test_simple` WHERE `id`=1;", "DELETE FROM `test`.`test_simple` WHERE `id`=2;", "DELETE FROM `test`.`test_simple` WHERE `id`=3;", "DELETE FROM `test`.`test_simple` WHERE `id`=4;", "DELETE FROM `test`.`test_simple` WHERE `id`=5;", "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(1,'a1',1),(2,'a2',2),(3,'a3',3),(4,'a4',4),(5,'a5',5);", ) } // TestThreadID 测试指定线程号,操作后断开重连 func (t *testParserSuite) TestGTID(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, `RESET MASTER;`, `DROP TABLE IF EXISTS test_simple`, `CREATE TABLE test_simple ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, c1 varchar(100), c2 int, PRIMARY KEY (id) ) ENGINE=InnoDB`) t.testExecute(c, `INSERT INTO test_simple (c1, c2) VALUES ('test',1)`, `DELETE FROM test_simple WHERE id > 0`) t.testExecute(c, `INSERT INTO test_simple (c1, c2) VALUES ('test2',2)`, `DELETE FROM test_simple WHERE id > 0`) uuid := t.getServerUUID(c) // ---- 错误GTID ---- t.SetIncludeGtids("123") _, err := core.NewBinlogParser(context.Background(), &t.config) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "错误GTID格式!正确格式为uuid:编号[-编号],多个时以逗号分隔") t.SetIncludeGtids(uuid) _, err = core.NewBinlogParser(context.Background(), &t.localConfig) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "错误GTID格式!正确格式为uuid:编号[-编号],多个时以逗号分隔") t.SetIncludeGtids(uuid + ":abc") _, err = core.NewBinlogParser(context.Background(), &t.localConfig) c.Assert(err, NotNil) c.Assert(err.Error(), Equals, "GTID解析失败!(strconv.ParseInt: parsing \"abc\": invalid syntax)") // ------ end ------ t.SetIncludeGtids(uuid + ":3") t.checkBinlog(c, "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(1,'test',1);") t.SetIncludeGtids(uuid + ":3-4") t.checkBinlog(c, "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(1,'test',1);", "DELETE FROM `test`.`test_simple` WHERE `id`=1;") t.SetIncludeGtids(uuid + ":5") t.checkBinlog(c, "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(2,'test2',2);") t.SetIncludeGtids(uuid + ":3-4," + uuid + ":6-7") t.checkBinlog(c, "INSERT INTO `test`.`test_simple`(`id`,`c1`,`c2`) VALUES(1,'test',1);", "DELETE FROM `test`.`test_simple` WHERE `id`=1;", "DELETE FROM `test`.`test_simple` WHERE `id`=2;", ) } func (t *testParserSuite) TestJson(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, `RESET MASTER;`, `DROP TABLE IF EXISTS test_json`, `CREATE TABLE test_json ( id BIGINT(64) UNSIGNED NOT NULL AUTO_INCREMENT, c1 JSON, c2 DECIMAL(10, 0), PRIMARY KEY (id) ) ENGINE=InnoDB`) t.testExecute(c, `INSERT INTO test_json (c2) VALUES (1)`, `INSERT INTO test_json (c1, c2) VALUES ('{"key1": "value1", "key2": "value2"}', 1)`, `update test_json set c1 = '{"key1": "value123"}',c2=2000 where id =2`, `delete from test_json where id > 0`) t.SetFlashback(true) t.checkBinlog(c, "DELETE FROM `test`.`test_json` WHERE `id`=1;", "DELETE FROM `test`.`test_json` WHERE `id`=2;", "UPDATE `test`.`test_json` SET `id`=2, `c1`='{\\\"key1\\\":\\\"value1\\\",\\\"key2\\\":\\\"value2\\\"}', `c2`=1 WHERE `id`=2;", "INSERT INTO `test`.`test_json`(`id`,`c1`,`c2`) VALUES(1,NULL,1);", "INSERT INTO `test`.`test_json`(`id`,`c1`,`c2`) VALUES(2,'{\\\"key1\\\":\\\"value123\\\"}',2000);") } func (t *testParserSuite) TestJsonV2(c *C) { t.setupTest(c, mysql.MySQLFlavor) t.testExecute(c, `RESET MASTER;`, "DROP TABLE IF EXISTS test_json_v2", `CREATE TABLE test_json_v2 ( id INT, c JSON, PRIMARY KEY (id) ) ENGINE=InnoDB`) tbls := []string{ `INSERT INTO test_json_v2 VALUES (0, NULL)`, `INSERT INTO test_json_v2 VALUES (1, '{\"a\": 2}')`, `INSERT INTO test_json_v2 VALUES (2, '[1,2]')`, `INSERT INTO test_json_v2 VALUES (3, '{\"a\":\"b\", \"c\":\"d\",\"ab\":\"abc\", \"bc\": [\"x\", \"y\"]}')`, `INSERT INTO test_json_v2 VALUES (4, '[\"here\", [\"I\", \"am\"], \"!!!\"]')`, `INSERT INTO test_json_v2 VALUES (5, '\"scalar string\"')`, `INSERT INTO test_json_v2 VALUES (6, 'true')`, `INSERT INTO test_json_v2 VALUES (7, 'false')`, `INSERT INTO test_json_v2 VALUES (8, 'null')`, `INSERT INTO test_json_v2 VALUES (9, '-1')`, `INSERT INTO test_json_v2 VALUES (10, CAST(CAST(1 AS UNSIGNED) AS JSON))`, `INSERT INTO test_json_v2 VALUES (11, '32767')`, `INSERT INTO test_json_v2 VALUES (12, '32768')`, `INSERT INTO test_json_v2 VALUES (13, '-32768')`, `INSERT INTO test_json_v2 VALUES (14, '-32769')`, `INSERT INTO test_json_v2 VALUES (15, '2147483647')`, `INSERT INTO test_json_v2 VALUES (16, '2147483648')`, `INSERT INTO test_json_v2 VALUES (17, '-2147483648')`, `INSERT INTO test_json_v2 VALUES (18, '-2147483649')`, `INSERT INTO test_json_v2 VALUES (19, '18446744073709551615')`, `INSERT INTO test_json_v2 VALUES (20, '18446744073709551616')`, `INSERT INTO test_json_v2 VALUES (21, '3.14')`, `INSERT INTO test_json_v2 VALUES (22, '{}')`, `INSERT INTO test_json_v2 VALUES (23, '[]')`, `INSERT INTO test_json_v2 VALUES (24, CAST(CAST('2015-01-15 23:24:25' AS DATETIME) AS JSON))`, `INSERT INTO test_json_v2 VALUES (25, CAST(CAST('23:24:25' AS TIME) AS JSON))`, `INSERT INTO test_json_v2 VALUES (125, CAST(CAST('23:24:25.12' AS TIME(3)) AS JSON))`, `INSERT INTO test_json_v2 VALUES (225, CAST(CAST('23:24:25.0237' AS TIME(3)) AS JSON))`, `INSERT INTO test_json_v2 VALUES (26, CAST(CAST('2015-01-15' AS DATE) AS JSON))`, `INSERT INTO test_json_v2 VALUES (27, CAST(TIMESTAMP'2015-01-15 23:24:25' AS JSON))`, `INSERT INTO test_json_v2 VALUES (127, CAST(TIMESTAMP'2015-01-15 23:24:25.12' AS JSON))`, `INSERT INTO test_json_v2 VALUES (227, CAST(TIMESTAMP'2015-01-15 23:24:25.0237' AS JSON))`, `INSERT INTO test_json_v2 VALUES (327, CAST(UNIX_TIMESTAMP('2015-01-15 23:24:25') AS JSON))`, `INSERT INTO test_json_v2 VALUES (28, CAST(ST_GeomFromText('POINT(1 1)') AS JSON))`, `INSERT INTO test_json_v2 VALUES (29, CAST('[]' AS CHAR CHARACTER SET 'ascii'))`, // TODO: 30 and 31 are BIT type from JSON_TYPE, may support later. `INSERT INTO test_json_v2 VALUES (30, CAST(x'cafe' AS JSON))`, `INSERT INTO test_json_v2 VALUES (31, CAST(x'cafebabe' AS JSON))`, `INSERT INTO test_json_v2 VALUES (100, CONCAT('{\"', REPEAT('a', 2 * 100 - 1), '\":123}'))`, } t.testExecute(c, tbls...) t.checkBinlog(c, "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(0,NULL);", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(1,'{\\\"a\\\":2}');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(2,'[1,2]');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(3,'{\\\"a\\\":\\\"b\\\",\\\"ab\\\":\\\"abc\\\",\\\"bc\\\":[\\\"x\\\",\\\"y\\\"],\\\"c\\\":\\\"d\\\"}');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(4,'[\\\"here\\\",[\\\"I\\\",\\\"am\\\"],\\\"!!!\\\"]');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(5,'\\\"scalar string\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(6,'true');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(7,'false');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(8,'null');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(9,'-1');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(10,'1');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(11,'32767');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(12,'32768');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(13,'-32768');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(14,'-32769');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(15,'2147483647');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(16,'2147483648');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(17,'-2147483648');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(18,'-2147483649');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(19,'18446744073709551615');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(20,'18446744073709552000');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(21,'3.14');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(22,'{}');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(23,'[]');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(24,'\\\"2015-01-15 23:24:25.000000\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(25,'\\\"23:24:25.000000\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(125,'\\\"23:24:25.120000\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(225,'\\\"23:24:25.024000\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(26,'\\\"2015-01-15 00:00:00.000000\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(27,'\\\"2015-01-15 23:24:25.000000\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(127,'\\\"2015-01-15 23:24:25.120000\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(227,'\\\"2015-01-15 23:24:25.023700\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(327,'1421335465');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(28,'{\\\"coordinates\\\":[1,1],\\\"type\\\":\\\"Point\\\"}');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(29,'[]');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(30,'\\\"\\\\ufffd\\\\ufffd\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(31,'\\\"\\\\ufffd\\\\ufffd\\\\ufffd\\\\ufffd\\\"');", "INSERT INTO `test`.`test_json_v2`(`id`,`c`) VALUES(100,'{\\\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\\\":123}');") t.testExecute(c, "delete from test_json_v2 where id >0") } func (t *testParserSuite) initTableSchema(tableName ...string) { var tables []string if len(tableName) > 0 { for _, name := range tableName { if v, ok := allTables[name]; ok { tables = append(tables, v) } } } else { for _, value := range allTables { tables = append(tables, value) } } err := ioutil.WriteFile(tableSchemaFile, []byte(strings.Join(tables, ";\n")), 0666) if err != nil { log.Fatal(err) } } // createTable 初始化时创建所有表 func (t *testParserSuite) createTables(c *C) { var tables []string for tableName, value := range allTables { tables = append(tables, value) tables = append(tables, fmt.Sprintf("truncate table `%s`;", tableName)) } t.testExecute(c, tables...) } func TestComputePercent(t *testing.T) { start := mysql.Position{ Name: "001", Pos: 0, } stop := mysql.Position{ Name: "001", Pos: 0, } current := mysql.Position{ Name: "001", Pos: 100, } master := core.MasterStatus{ File: "002", Position: 600, } binlogs := []core.MasterLog{ { Name: "001", Size: 1000, }, { Name: "002", Size: 1000, }, } pct := core.ComputePercent(start, stop, current, master, binlogs) expected := 10 if pct != expected { t.Errorf("got %v expected %v", pct, expected) } stop = mysql.Position{ Name: "002", Pos: 0, } pct = core.ComputePercent(start, stop, current, master, binlogs) expected = 5 if pct != expected { t.Errorf("got %v expected %v", pct, expected) } stop = mysql.Position{ Name: "003", Pos: 0, } pct = core.ComputePercent(start, stop, current, master, binlogs) expected = 5 if pct != expected { t.Errorf("got %v expected %v", pct, expected) } start = mysql.Position{ Name: "001", Pos: 200, } stop = mysql.Position{ Name: "", Pos: 0, } current = mysql.Position{ Name: "002", Pos: 400, } pct = core.ComputePercent(start, stop, current, master, binlogs) expected = 85 if pct != expected { t.Errorf("got %v expected %v", pct, expected) } } ================================================ FILE: core/socket.go ================================================ // go-mysqlbinlog: a simple binlog tool to sync remote MySQL binlog. // go-mysqlbinlog supports semi-sync mode like facebook mysqlbinlog. // see http://yoshinorimatsunobu.blogspot.com/2014/04/semi-synchronous-replication-at-facebook.html package core // time go run mainRemote.go -start-time="2018-09-17 00:00:00" -stop-time="2018-09-25 00:00:00" -o=1.sql import ( "fmt" "github.com/imroc/req" log "github.com/sirupsen/logrus" ) var URL string var header = req.Header{"Accept": "application/json"} // func init() { // cnf, err := ini.Load("../cnf/config.ini") // if err != nil { // log.Fatal().Err(err).Msg("加载配置文件失败!") // return // } // addr := cnf.Section("Bingo").Key("socketAddr").String() // URL = fmt.Sprintf("http://%s/socket", addr) // req.SetTimeout(5 * time.Second) // } // 向客户端推送消息 func sendMsg(user string, event string, title string, text string, kwargs map[string]interface{}) bool { if user == "" { return true } if URL == "" { return true } url := fmt.Sprintf("%s/room/%s", URL, user) param := req.Param{ "event": event, "title": title, "content": text, } if kwargs != nil && len(kwargs) > 0 { for k, v := range kwargs { param[k] = v } } r, err := req.Post(url, header, param) if err != nil { log.Error("请求websocket失败!") log.Print(err) return false } // r.ToJSON(&foo) // 响应体转成对象 // log.Printf("%+v", r) // 打印详细信息 resp := r.Response() if resp.StatusCode == 200 { return true } else { log.Error("请求websocket失败!") log.Printf("%+v", r) // 打印详细信息 return false } } ================================================ FILE: core/time.go ================================================ package core import ( "time" ) func timeTrack(start time.Time, name string) { // elapsed := time.Since(start) // log.Printf("%s took %s", name, time.Since(start)) } ================================================ FILE: docs/test.md ================================================ # 效率测试 测试结果 mysqlbinlog > bingo2sql > binlog_rollback > binlog2sql **说明：** 该测试结果可能存在的偏差如下： - 输出格式略有差异 - 测试所在环境的状态变化 - binlog_rollback 输出了binlog分析等信息等 ### 测试数据准备 - 准备100w行基础数据 - 更新 - 删除 - 获取binlog位置 #### 创建测试表 ```sql use test; reset master; drop table if exists tt; CREATE TABLE `tt` ( ID bigint unsigned auto_increment primary key, `TABLE_CATALOG` varchar(512) NOT NULL DEFAULT '', `TABLE_SCHEMA` varchar(64) NOT NULL DEFAULT '', `TABLE_NAME` varchar(64) NOT NULL DEFAULT '', `COLUMN_NAME` varchar(64) NOT NULL DEFAULT '', `ORDINAL_POSITION` bigint(21) unsigned NOT NULL DEFAULT '0', `COLUMN_DEFAULT` longtext, `IS_NULLABLE` varchar(3) NOT NULL DEFAULT '', `DATA_TYPE` varchar(64) NOT NULL DEFAULT '', `CHARACTER_MAXIMUM_LENGTH` bigint(21) unsigned DEFAULT NULL, `CHARACTER_OCTET_LENGTH` bigint(21) unsigned DEFAULT NULL, `NUMERIC_PRECISION` bigint(21) unsigned DEFAULT NULL, `NUMERIC_SCALE` bigint(21) unsigned DEFAULT NULL, `DATETIME_PRECISION` bigint(21) unsigned DEFAULT NULL, `CHARACTER_SET_NAME` varchar(32) DEFAULT NULL, `COLLATION_NAME` varchar(32) DEFAULT NULL, `COLUMN_TYPE` longtext NOT NULL, `COLUMN_KEY` varchar(3) NOT NULL DEFAULT '', `EXTRA` varchar(30) NOT NULL DEFAULT '', `PRIVILEGES` varchar(80) NOT NULL DEFAULT '', `COLUMN_COMMENT` varchar(1024) NOT NULL DEFAULT '', `GENERATION_EXPRESSION` longtext NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8; ``` #### 搭建测试数据 ```sql insert into tt(TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,ORDINAL_POSITION,COLUMN_DEFAULT,IS_NULLABLE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH,CHARACTER_OCTET_LENGTH,NUMERIC_PRECISION,NUMERIC_SCALE,DATETIME_PRECISION,CHARACTER_SET_NAME,COLLATION_NAME,COLUMN_TYPE,COLUMN_KEY,EXTRA,PRIVILEGES,COLUMN_COMMENT,GENERATION_EXPRESSION) select * from information_schema.columns limit 1000; # 准备100w行基础数据 insert into tt(TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,ORDINAL_POSITION,COLUMN_DEFAULT,IS_NULLABLE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH,CHARACTER_OCTET_LENGTH,NUMERIC_PRECISION,NUMERIC_SCALE,DATETIME_PRECISION,CHARACTER_SET_NAME,COLLATION_NAME,COLUMN_TYPE,COLUMN_KEY,EXTRA,PRIVILEGES,COLUMN_COMMENT,GENERATION_EXPRESSION) select tt.TABLE_CATALOG,tt.TABLE_SCHEMA,tt.TABLE_NAME,tt.COLUMN_NAME,tt.ORDINAL_POSITION,tt.COLUMN_DEFAULT,tt.IS_NULLABLE,tt.DATA_TYPE,tt.CHARACTER_MAXIMUM_LENGTH,tt.CHARACTER_OCTET_LENGTH,tt.NUMERIC_PRECISION,tt.NUMERIC_SCALE,tt.DATETIME_PRECISION,tt.CHARACTER_SET_NAME,tt.COLLATION_NAME,tt.COLUMN_TYPE,tt.COLUMN_KEY,tt.EXTRA,tt.PRIVILEGES,tt.COLUMN_COMMENT,tt.GENERATION_EXPRESSION from tt,tt t1; update tt set ORDINAL_POSITION=ORDINAL_POSITION+10 where id%5=0; update tt set TABLE_NAME=concat(TABLE_NAME,"1"),COLLATION_NAME=concat(COLLATION_NAME,"2"), COLUMN_COMMENT = TABLE_NAME where id%4=0; delete from tt where id%2=0; # 获取binlog位置以便设置解析参数 show BINARY LOGS; ``` Log_name | File_size -----------| ----------- mysql-bin.000001 | 357491554 #### bin2sql 测试脚本 ```sh for i in {1..10}; do echo $i; /usr/bin/time -f "real %E | user %U | sys %S | cpu %P" -a -o out.txt bin/bingo2sql -h 127.0.0.1 -P 3306 -u test -p test --start-file=mysql-bin.000001 --start-pos=4 --stop-file=mysql-bin.000001 --stop-pos=357490000 -o /tmp/binlog2/out.sql --max=0 --show-gtid=false --show-time=false; done cat out.txt; ``` 执行结果 real | user | sys | cpu ---- | ---- | ---- | ---- real 0:13.49 | user 33.65 | sys 2.68 | cpu 269% real 0:13.61 | user 34.21 | sys 2.75 | cpu 271% real 0:13.42 | user 33.26 | sys 2.61 | cpu 267% real 0:13.61 | user 33.99 | sys 2.64 | cpu 269% real 0:13.85 | user 34.77 | sys 2.65 | cpu 270% real 0:13.58 | user 33.58 | sys 2.62 | cpu 266% real 0:13.61 | user 34.06 | sys 2.59 | cpu 269% real 0:13.47 | user 33.39 | sys 2.65 | cpu 267% real 0:11.68 | user 28.75 | sys 2.29 | cpu 265% real 0:11.45 | user 28.68 | sys 2.11 | cpu 268% #### binlog_rollback 测试脚本 ```sh rm -f out.txt; for i in {1..10}; do echo $i; /usr/bin/time -f "real %E | user %U | sys %S | cpu %P" -a -o out.txt ./binlog_rollback -m repl -w 2sql -M mysql -t 4 -H 127.0.0.1 -P 3306 -u test -p test -sbin mysql-bin.000001 -spos 4 -ebin mysql-bin.000001 -epos 357490000 -o /tmp/binlog -dj="" done cat out.txt; ``` 执行结果 real | user | sys | cpu ---- | ---- | ---- | ---- real 0:28.79 | user 68.74 | sys 8.46 | cpu 268% real 0:27.35 | user 66.22 | sys 7.86 | cpu 270% real 0:28.58 | user 69.29 | sys 8.31 | cpu 271% real 0:29.19 | user 70.78 | sys 8.47 | cpu 271% real 0:28.31 | user 68.91 | sys 8.06 | cpu 271% real 0:28.67 | user 68.67 | sys 8.33 | cpu 268% real 0:28.40 | user 68.92 | sys 7.73 | cpu 269% real 0:28.44 | user 69.04 | sys 8.05 | cpu 271% real 0:28.43 | user 68.97 | sys 7.94 | cpu 270% real 0:28.87 | user 69.27 | sys 7.99 | cpu 267% #### bin2sql测试脚本 ```sh rm -f out.txt; for i in {1..10}; do echo $i; /usr/bin/time -f "real %E | user %U | sys %S | cpu %P" -a -o out.txt python binlog2sql.py -h127.0.0.1 -P3306 -utest -p'test' --start-file='mysql-bin.000001' --start-position=4 --stop-file='mysql-bin.000001' --stop-position=357490000 --only-dml > /tmp/binlog3/out.sql done cat out.txt; ``` 执行结果 **无** 感兴趣的同学可以自行测试。 #### mysqlbinlog 测试脚本 ```sh rm -f out.txt; for i in {1..10}; do echo $i; /usr/bin/time -f "real %E | user %U | sys %S | cpu %P" -a -o out.txt mysqlbinlog -h127.0.0.1 -P3306 -utest -p'test' --start-position=4 --stop-position=357490000 --base64-output=DECODE-ROWS -v -r /tmp/binlog4/out.sql /data/mysql/db_cmdb/blog/mysql-bin.000001 done cat out.txt; ``` 执行结果 real | user | sys | cpu ---- | ---- | ---- | ---- real 0:13.42 | user 11.43 | sys 1.96 | cpu 99% real 0:13.29 | user 11.37 | sys 1.88 | cpu 99% real 0:13.59 | user 11.54 | sys 1.94 | cpu 99% real 0:13.17 | user 11.20 | sys 1.94 | cpu 99% real 0:13.27 | user 11.37 | sys 1.83 | cpu 99% real 0:13.76 | user 11.62 | sys 2.04 | cpu 99% real 0:13.98 | user 12.01 | sys 1.93 | cpu 99% real 0:14.05 | user 11.86 | sys 1.95 | cpu 98% real 0:13.76 | user 11.67 | sys 1.96 | cpu 99% real 0:13.79 | user 11.72 | sys 2.01 | cpu 99% ================================================ FILE: go.mod ================================================ module github.com/hanchuanchuan/bingo2sql replace github.com/sirupsen/logrus => github.com/sirupsen/logrus v1.2.0 go 1.18 // replace golang.org/x/sys => github.com/golang/sys v0.0.0-20200201011859-915c9c3d4ccf require ( github.com/go-mysql-org/go-mysql v1.6.0 github.com/go-sql-driver/mysql v1.5.0 github.com/hanchuanchuan/goInception v1.2.0 github.com/imroc/req v0.3.0 github.com/jinzhu/gorm v1.9.12 github.com/jinzhu/now v1.1.1 github.com/labstack/echo/v4 v4.1.14 github.com/mholt/archiver/v3 v3.3.0 github.com/mitchellh/go-homedir v1.1.0 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 github.com/pkg/profile v1.4.0 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.6 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.6.2 ) require github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a // indirect require ( github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6 // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/frankban/quicktest v1.7.2 // indirect github.com/fsnotify/fsnotify v1.4.7 // indirect github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 // indirect github.com/golang/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.1 // indirect github.com/google/go-cmp v0.3.1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 // indirect github.com/klauspost/compress v1.9.7 // indirect github.com/klauspost/pgzip v1.2.1 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/labstack/gommon v0.3.0 // indirect github.com/magiconair/properties v1.8.1 // indirect github.com/mattn/go-colorable v0.1.4 // indirect github.com/mattn/go-isatty v0.0.11 // indirect github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/nwaples/rardecode v1.0.0 // indirect github.com/pelletier/go-toml v1.6.0 // indirect github.com/pierrec/lz4 v2.4.1+incompatible // indirect github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 // indirect github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 // indirect github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cast v1.3.1 // indirect // github.com/spf13/cobra v0.0.6 github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/ulikunitz/xz v0.5.6 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.1.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/youtube/vitess v2.1.1+incompatible // indirect go.uber.org/atomic v1.7.0 // indirect golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect golang.org/x/sys v0.0.0-20220624220833-87e55d714810 // indirect golang.org/x/text v0.3.6 // indirect gopkg.in/ini.v1 v1.52.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect ) ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/CorgiMan/json2 v0.0.0-20150213135156-e72957aba209 h1:rRZPHlNlREFwuEtpYikMNZPs4l5g6zic54l2XDAK4ws= github.com/CorgiMan/json2 v0.0.0-20150213135156-e72957aba209/go.mod h1:VwmVPvMIZlx23Q7F1umyYmkhNDqf6WQKfMUhGEdVcLA= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6 h1:bZ28Hqta7TFAK3Q08CMvv8y3/8ATaEqv2nGoc6yff6c= github.com/andybalholm/brotli v0.0.0-20190621154722-5f990b63d2d6/go.mod h1:+lx6/Aqd1kLJ1GQfkvOnaZ1WGmLpMpbprPuIOOZX30U= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/blacktear23/go-proxyprotocol v0.0.0-20171102103907-62e368e1c470/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/bbolt v1.3.0/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cznic/golex v0.0.0-20181122101858-9c343928389c/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= github.com/cznic/mathutil v0.0.0-20181021201202-eba54fb065b7/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/cznic/parser v0.0.0-20160622100904-31edd927e5b1/go.mod h1:2B43mz36vGZNZEwkWi8ayRSSUXLfjL8OkbzwW4NcPMM= github.com/cznic/parser v0.0.0-20181122101858-d773202d5b1f/go.mod h1:2B43mz36vGZNZEwkWi8ayRSSUXLfjL8OkbzwW4NcPMM= github.com/cznic/sortutil v0.0.0-20150617083342-4c7342852e65/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 h1:LpMLYGyy67BoAFGda1NeOBQwqlv7nUXpm+rIVHGxZZ4= github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ= github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= github.com/cznic/y v0.0.0-20170802143616-045f81c6662a/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= github.com/cznic/y v0.0.0-20181122101901-b05e8c2e8d7b/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20190121005146-b04fd42d9952/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a h1:QNEenQIsGDEEfFNSnN+h6hE1OwnHqTg7Dl9gEk1Cko4= github.com/etcd-io/gofail v0.0.0-20180808172546-51ce9a71510a/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-mysql-org/go-mysql v1.6.0 h1:19B5fojzZcri/1wj9G/1+ws8RJ3N6rJs2X5c/+kBLuQ= github.com/go-mysql-org/go-mysql v1.6.0/go.mod h1:GX0clmylJLdZEYAojPCDTCvwZxbTBrke93dV55715u0= github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 h1:KRMr9A3qfbVM7iV/WcLY/rL5LICqwMHLhwRXKu99fXw= github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.5.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hanchuanchuan/gh-ost v1.0.49-0.20200114083508-62a578b91654 h1:PRc0l9OaUyPNPoMo0nruucADDwpFZkVvxMbGCZXpjFg= github.com/hanchuanchuan/gh-ost v1.0.49-0.20200114083508-62a578b91654/go.mod h1:9NockcUOKVL+JLrnCsXtpRAuPseKoMVqfDmn251PbG0= github.com/hanchuanchuan/go-mysql v0.0.0-20200114082439-6d0d8d3a982e h1:IjLlaPxsWKkYuePu8Ed2YbHLuBAuKLW6lclhmeq2A6I= github.com/hanchuanchuan/go-mysql v0.0.0-20200114082439-6d0d8d3a982e/go.mod h1:HJV3Ej7p/Ck/htb5F5VwYAHDL02jA4J6uAufArtmsos= github.com/hanchuanchuan/goInception v1.2.0 h1:uAai3jAT8D7zwE6YhmMDaJg+3eTPfQauYiNMq9xtztY= github.com/hanchuanchuan/goInception v1.2.0/go.mod h1:idDo5pLO0j10hNjc2xf1fht01QEbUNTTyPVHGdDp9Z4= github.com/hanchuanchuan/golib v0.0.0-20200113085747-47643bc243f1 h1:h4yTn7Lu9roQdSZezhf8BwMi3R/KWikL6xV02rmZj6Q= github.com/hanchuanchuan/golib v0.0.0-20200113085747-47643bc243f1/go.mod h1:6V4x6V71pCU4o9UfPmemdrU9sEhSF6AmXt+PiFr8x4Q= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imroc/req v0.2.3/go.mod h1:J9FsaNHDTIVyW/b5r6/Df5qKEEEq2WzZKIgKSajd1AE= github.com/imroc/req v0.3.0 h1:3EioagmlSG+z+KySToa+Ylo3pTFZs+jh3Brl7ngU12U= github.com/imroc/req v0.3.0/go.mod h1:F+NZ+2EFSo6EFXdeIbpfE9hcC233id70kf0byW97Caw= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jinzhu/gorm v1.9.2/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v0.0.0-20181116074157-8ec929ed50c3/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6Gfc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9 h1:hJix6idebFclqlfZCHE7EUX7uqLCyb70nHNHH1XKGBg= github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7 h1:hYW1gP94JUmAhBtJ+LNz5My+gBobDxPR1iVuKug26aA= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labstack/echo/v4 v4.1.14 h1:h8XP66UfB3tUm+L3QPw7tmwAu3pJaA/nyfHPCcz46ic= github.com/labstack/echo/v4 v4.1.14/go.mod h1:Q5KZ1vD3V5FEzjM79hjwVrC3ABr7F5IdM23bXQMRDGg= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mholt/archiver/v3 v3.3.0 h1:vWjhY8SQp5yzM9P6OJ/eZEkmi3UAbRrxCq48MxjAzig= github.com/mholt/archiver/v3 v3.3.0/go.mod h1:YnQtqsp+94Rwd0D/rk5cnLrxusUBUXg+08Ebtr1Mqao= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/montanaflynn/stats v0.0.0-20180911141734-db72e6cae808/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/myesui/uuid v1.0.0/go.mod h1:2CDfNgU0LR8mIdO8vdWd8i9gWWxLlcoIGGpSNgafq84= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs= github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/outbrain/golib v0.0.0-20180830062331-ab954725f502 h1:oS2s2j8GP70jE0IDMyA4BO4HgvkNcjHfR40h2fyhc7s= github.com/outbrain/golib v0.0.0-20180830062331-ab954725f502/go.mod h1:JDhu//MMvcPVPH889Xr7DyamEbTLumgDBALGUyXrz1g= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/percona/go-mysql v0.0.0-20190307200310-f5cfaf6a5e55 h1:kx48fD4K+GXb7YqwykiWirM8GRtoMcpWbGlaAb3IsqE= github.com/percona/go-mysql v0.0.0-20190307200310-f5cfaf6a5e55/go.mod h1:/SGLf9OMxlnK6jq4mkFiImBcJXXk5jwD+lDrwDaGXcw= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI= github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 h1:LllgC9eGfqzkfubMgjKIDyZYaa609nNWAyNZtpy2B3M= github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rGrobssy1nVy2VaVpNCuLpCbr+FEaTA8= github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20181206061346-54cf0a0dfe55 h1:0vbRC39OU0Ep2GFgV5BwZ1u3viC5uk8yb3c07rLRMvY= github.com/pingcap/kvproto v0.0.0-20181206061346-54cf0a0dfe55/go.mod h1:Ja9XPjot9q4/3JyCZodnWDGNXt4pKemhIYCvVJM7P24= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/parser v0.0.0-20210415081931-48e7f467fd74/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw= github.com/pingcap/pd v2.1.0+incompatible h1:X0o443C/jXF6yJiiP1xTRxjKPHRe4gwQqjGcFTz1MuU= github.com/pingcap/pd v2.1.0+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/tipb v0.0.0-20170310053819-1043caee48da/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330 h1:rRMLMjIMFulCX9sGKZ1hoov/iROMsKyC8Snc02nSukw= github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.4.0 h1:uCmaf4vVbWAOZz36k1hrQD7ijGRzLwaME8Am/7a4jZI= github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 h1:HQagqIiBmr8YXawX/le3+O26N+vPPC1PtjaF3mwnook= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.6 h1:breEStsVwemnKh2/s6gMvSdMEkwW0sK8vGStnlVBMCs= github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twinj/uuid v1.0.0 h1:fzz7COZnDrXGTAOHGuUGYd6sG+JMq+AoE7+Jlu0przk= github.com/twinj/uuid v1.0.0/go.mod h1:mMgcE1RHFUFqe5AfiwlINXisXfDGro23fWdPUfOMjRY= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1C1PjvOJnJykCzcD5QHbk= github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/unrolled/render v0.0.0-20180914162206-b9786414de4d/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youtube/vitess v2.1.1+incompatible h1:SE+P7DNX/jw5RHFs5CHRhZQjq402EJFCD33JhzQMdDw= github.com/youtube/vitess v2.1.1+incompatible/go.mod h1:hpMim5/30F1r+0P8GGtB29d0gWHr0IZ5unS+CG0zMx8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220624220833-87e55d714810 h1:rHZQSjJdAI4Xf5Qzeh2bBc5YJIkPFVM6oDtMFYmgws0= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.0.0-20171214130843-f21a4dfb5e38/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181008205924-a2b3f7f249e9/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f h1:FU37niK8AQ59mHcskRyQL7H0ErSeNh650vdcj8HqdSI= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.52.0 h1:j+Lt/M1oPPejkniCg1TkWE2J3Eh1oZTsHSXzMTzUXn4= gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/stretchr/testify.v1 v1.2.2/go.mod h1:QI5V/q6UbPmuhtm10CaFZxED9NreB8PnFYN9JcR6TxU= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= vitess.io/vitess v2.1.1+incompatible h1:nuuGHiWYWpudD3gOCLeGzol2EJ25e/u5Wer2wV1O130= vitess.io/vitess v2.1.1+incompatible/go.mod h1:h4qvkyNYTOC0xI+vcidSWoka0gQAZc9ZPHbkHo48gP0= ================================================ FILE: main.go ================================================ package main import "github.com/hanchuanchuan/bingo2sql/cmd" func main() { cmd.Execute() } ================================================ FILE: parse/bingo2sql.go ================================================ package parse import ( "context" "fmt" "net/http" "os" "sync" "time" "github.com/hanchuanchuan/bingo2sql/core" _ "github.com/jinzhu/gorm/dialects/mysql" "github.com/labstack/echo/v4" log "github.com/sirupsen/logrus" ) // var parserProcess map[string]*parser.MyBinlogParser var parserProcess sync.Map type ParseInfo struct { ID string `json:"id"` ParseRows int `json:"parse_rows"` Percent int `json:"percent"` } func TestMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Do stuff here fmt.Println("middleware print: ", r.RequestURI) // Call the next handler, which can be another middleware in the chain, or the final handler. next.ServeHTTP(w, r) }) } func HomeHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, "this is home") } // GetParseInfo 获取解析进程信息 func GetParseInfo(c echo.Context) error { id := c.Param("id") if len(id) == 0 { return c.JSON(http.StatusOK, map[string]string{ "error": "无效参数!", }) } if v, ok := parserProcess.Load(id); ok { if p, ok := v.(*core.MyBinlogParser); ok { data := &ParseInfo{ ID: id, ParseRows: p.ParseRows(), Percent: p.Percent(), } return c.JSON(http.StatusOK, data) } } return c.JSON(http.StatusNotFound, "Not Found!") } // GetAllParse 获取所有进程信息 func GetAllParse(c echo.Context) error { // log.Print("当前解析进程数量: ", len(parserProcess.Range())) // // for _, p := range parserProcess { // data := &ParseInfo{ // ID: p.Config().Id(), // ParseRows: p.ParseRows(), // Percent: p.Percent(), // } // response = append(response, data) // } // return c.JSON(http.StatusOK, response) count := 0 // response := make([]*ParseInfo, 0, len(parserProcess)) response := make([]*ParseInfo, 0) // 遍历所有sync.Map中的键值对 parserProcess.Range(func(k, v interface{}) bool { count++ if p, ok := v.(*core.MyBinlogParser); ok { data := &ParseInfo{ ID: p.Config().ID(), ParseRows: p.ParseRows(), Percent: p.Percent(), } response = append(response, data) } fmt.Println("iterate:", k, v) return true }) log.Print("当前解析进程数量: ", count) return c.JSON(http.StatusOK, response) } func ParseBinlog(c echo.Context) error { cfg := &core.BinlogParserConfig{ ShowGTID: true, ShowTime: true, MinimalInsert: true, MinimalUpdate: true, } if err := c.Bind(cfg); err != nil { return err } log.Infof("config: %#v", cfg) // 指定默认的socket用户,用来生成SQL文件 if cfg.SocketUser == "" { cfg.SocketUser = "test" } // if cfg.InsID == 0 { // r := map[string]string{"error": "请指定数据库地址"} // return c.JSON(http.StatusOK, r) // } p, err := core.NewBinlogParser(context.Background(), cfg) if err != nil { log.Error("binlog解析操作失败") log.Error(err) out := map[string]string{"error": err.Error()} return c.JSON(http.StatusOK, out) } if err := recover(); err != nil { if e, ok := err.(error); ok { log.Error("binlog解析操作失败") log.Error(err) out := map[string]string{"error": e.Error()} return c.JSON(http.StatusOK, out) } else { out := map[string]string{"error": "未知的错误"} return c.JSON(http.StatusOK, out) } } else { id := cfg.ID() // parserProcess[id] = p parserProcess.Store(id, p) go func() { defer func() { time.AfterFunc(time.Minute, func() { parserProcess.Delete(id) }) }() err := p.Parser() if err != nil { log.Error(err) } }() r := map[string]string{"id": id} return c.JSON(http.StatusOK, r) } } func ParseBinlogStop(c echo.Context) error { id := c.Param("id") if len(id) == 0 { return c.JSON(http.StatusOK, map[string]string{ "error": "无效参数!", }) } response := make(map[string]string) // log.Print("当前解析进程数量: ", len(parserProcess)) // defer delete(parserProcess, id) defer parserProcess.Delete(id) if v, ok := parserProcess.Load(id); ok { if p, ok := v.(*core.MyBinlogParser); ok { p.Stop() response["ok"] = "1" return c.JSON(http.StatusOK, response) } } response["ok"] = "2" return c.JSON(http.StatusOK, response) // if p, ok := parserProcess[id]; ok { // p.Stop() // response["ok"] = "1" // return c.JSON(http.StatusOK, response) // } else { // response["ok"] = "2" // return c.JSON(http.StatusOK, response) // } } func Download(c echo.Context) error { id := c.Param("id") path := "files/" + id + ".tar.gz" _, err := os.Stat(path) if err != nil { log.Error(err) return c.JSON(http.StatusNotFound, fmt.Sprintf("%s no such file or directory", id)) // return err } else { log.Infof("下载路径: %s", path) return c.Attachment(path, c.Param("name")) } // return c.Inline(path, c.Param("name")) // return c.File(path) } ================================================ FILE: parse/log.go ================================================ package parse import ( "fmt" "path" "runtime" "strings" log "github.com/sirupsen/logrus" ) var ( // BuildTS 打包时间 BuildTS = "" // GitHash GIT提交ID GitHash = "" // GitBranch GIT分支 GitBranch = "" // GoVersion GOLANG版本 GoVersion = "" // GitVersion GIT版本 GitVersion = "" ) func init() { log.SetFormatter(&log.TextFormatter{ DisableColors: false, FullTimestamp: true, }) // log.SetFormatter(&log.TextFormatter{ // DisableColors: true, // FullTimestamp: true, // }) // log.SetReportCaller(true) // 输出文件名和行号 log.AddHook(&contextHook{}) } // modifyHook injects file name and line pos into log entry. type contextHook struct{} // Fire implements logrus.Hook interface // https://github.com/sirupsen/logrus/issues/63 func (hook *contextHook) Fire(entry *log.Entry) error { pc := make([]uintptr, 4) cnt := runtime.Callers(6, pc) for i := 0; i < cnt; i++ { fu := runtime.FuncForPC(pc[i] - 1) name := fu.Name() if !isSkippedPackageName(name) { file, line := fu.FileLine(pc[i] - 1) entry.Data["file"] = path.Base(file) entry.Data["line"] = line break } } return nil } // Levels implements logrus.Hook interface. func (hook *contextHook) Levels() []log.Level { return log.AllLevels } // isSKippedPackageName tests wether path name is on log library calling stack. func isSkippedPackageName(name string) bool { return strings.Contains(name, "github.com/sirupsen/logrus") || strings.Contains(name, "github.com/coreos/pkg/capnslog") } // PrintVersion prints the version information. func PrintVersion() { log.Infof("Version: %s", GitVersion) log.Infof("Git Commit Hash: %s", GitHash) log.Infof("Git Branch: %s", GitBranch) log.Infof("UTC Build Time: %s", BuildTS) log.Infof("Go Version: %s", GoVersion) } // GetInfo returns the git hash and build time of this -server binary. func GetInfo() string { return fmt.Sprintf( "Version: %v\n"+ "Git Commit Hash: %s\n"+ "Git Branch: %s\n"+ "UTC Build Time: %s\n"+ "Go Version: %s", GitVersion, GitHash, GitBranch, BuildTS, GoVersion, ) } ================================================ FILE: utils/uuid/codec.go ================================================ // Copyright (C) 2013-2018 by Maxim Bublis // // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including // without limitation the rights to use, copy, modify, merge, publish, // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: // // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. package uuid import ( "bytes" "encoding/hex" "fmt" ) // FromBytes returns UUID converted from raw byte slice input. // It will return error if the slice isn't 16 bytes long. func FromBytes(input []byte) (u UUID, err error) { err = u.UnmarshalBinary(input) return } // FromBytesOrNil returns UUID converted from raw byte slice input. // Same behavior as FromBytes, but returns a Nil UUID on error. func FromBytesOrNil(input []byte) UUID { uuid, err := FromBytes(input) if err != nil { return Nil } return uuid } // FromString returns UUID parsed from string input. // Input is expected in a form accepted by UnmarshalText. func FromString(input string) (u UUID, err error) { err = u.UnmarshalText([]byte(input)) return } // FromStringOrNil returns UUID parsed from string input. // Same behavior as FromString, but returns a Nil UUID on error. func FromStringOrNil(input string) UUID { uuid, err := FromString(input) if err != nil { return Nil } return uuid } // MarshalText implements the encoding.TextMarshaler interface. // The encoding is the same as returned by String. func (u UUID) MarshalText() (text []byte, err error) { text = []byte(u.String()) return } // UnmarshalText implements the encoding.TextUnmarshaler interface. // Following formats are supported: // "6ba7b810-9dad-11d1-80b4-00c04fd430c8", // "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}", // "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" // "6ba7b8109dad11d180b400c04fd430c8" // ABNF for supported UUID text representation follows: // uuid := canonical | hashlike | braced | urn // plain := canonical | hashlike // canonical := 4hexoct '-' 2hexoct '-' 2hexoct '-' 6hexoct // hashlike := 12hexoct // braced := '{' plain '}' // urn := URN ':' UUID-NID ':' plain // URN := 'urn' // UUID-NID := 'uuid' // 12hexoct := 6hexoct 6hexoct // 6hexoct := 4hexoct 2hexoct // 4hexoct := 2hexoct 2hexoct // 2hexoct := hexoct hexoct // hexoct := hexdig hexdig // hexdig := '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | // 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | // 'A' | 'B' | 'C' | 'D' | 'E' | 'F' func (u *UUID) UnmarshalText(text []byte) (err error) { switch len(text) { case 32: return u.decodeHashLike(text) case 36: return u.decodeCanonical(text) case 38: return u.decodeBraced(text) case 41: fallthrough case 45: return u.decodeURN(text) default: return fmt.Errorf("uuid: incorrect UUID length: %s", text) } } // decodeCanonical decodes UUID string in format // "6ba7b810-9dad-11d1-80b4-00c04fd430c8". func (u *UUID) decodeCanonical(t []byte) (err error) { if t[8] != '-' || t[13] != '-' || t[18] != '-' || t[23] != '-' { return fmt.Errorf("uuid: incorrect UUID format %s", t) } src := t[:] dst := u[:] for i, byteGroup := range byteGroups { if i > 0 { src = src[1:] // skip dash } _, err = hex.Decode(dst[:byteGroup/2], src[:byteGroup]) if err != nil { return } src = src[byteGroup:] dst = dst[byteGroup/2:] } return } // decodeHashLike decodes UUID string in format // "6ba7b8109dad11d180b400c04fd430c8". func (u *UUID) decodeHashLike(t []byte) (err error) { src := t[:] dst := u[:] if _, err = hex.Decode(dst, src); err != nil { return err } return } // decodeBraced decodes UUID string in format // "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}" or in format // "{6ba7b8109dad11d180b400c04fd430c8}". func (u *UUID) decodeBraced(t []byte) (err error) { l := len(t) if t[0] != '{' || t[l-1] != '}' { return fmt.Errorf("uuid: incorrect UUID format %s", t) } return u.decodePlain(t[1 : l-1]) } // decodeURN decodes UUID string in format // "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8" or in format // "urn:uuid:6ba7b8109dad11d180b400c04fd430c8". func (u *UUID) decodeURN(t []byte) (err error) { total := len(t) urn_uuid_prefix := t[:9] if !bytes.Equal(urn_uuid_prefix, urnPrefix) { return fmt.Errorf("uuid: incorrect UUID format: %s", t) } return u.decodePlain(t[9:total]) } // decodePlain decodes UUID string in canonical format // "6ba7b810-9dad-11d1-80b4-00c04fd430c8" or in hash-like format // "6ba7b8109dad11d180b400c04fd430c8". func (u *UUID) decodePlain(t []byte) (err error) { switch len(t) { case 32: return u.decodeHashLike(t) case 36: return u.decodeCanonical(t) default: return fmt.Errorf("uuid: incorrrect UUID length: %s", t) } } // MarshalBinary implements the encoding.BinaryMarshaler interface. func (u UUID) MarshalBinary() (data []byte, err error) { data = u.Bytes() return } // UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. // It will return error if the slice isn't 16 bytes long. func (u *UUID) UnmarshalBinary(data []byte) (err error) { if len(data) != Size { err = fmt.Errorf("uuid: UUID must be exactly 16 bytes long, got %d bytes", len(data)) return } copy(u[:], data) return } ================================================ FILE: utils/uuid/generator.go ================================================ // Copyright (C) 2013-2018 by Maxim Bublis // // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including // without limitation the rights to use, copy, modify, merge, publish, // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: // // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. package uuid import ( "crypto/md5" "crypto/rand" "crypto/sha1" "encoding/binary" "fmt" "hash" "io" "net" "os" "sync" "time" ) // Difference in 100-nanosecond intervals between // UUID epoch (October 15, 1582) and Unix epoch (January 1, 1970). const epochStart = 122192928000000000 type epochFunc func() time.Time type hwAddrFunc func() (net.HardwareAddr, error) var ( global = newRFC4122Generator() posixUID = uint32(os.Getuid()) posixGID = uint32(os.Getgid()) ) // NewV1 returns UUID based on current timestamp and MAC address. func NewV1() (UUID, error) { return global.NewV1() } // NewV2 returns DCE Security UUID based on POSIX UID/GID. func NewV2(domain byte) (UUID, error) { return global.NewV2(domain) } // NewV3 returns UUID based on MD5 hash of namespace UUID and name. func NewV3(ns UUID, name string) UUID { return global.NewV3(ns, name) } // NewV4 returns random generated UUID. func NewV4() (UUID, error) { return global.NewV4() } // NewV5 returns UUID based on SHA-1 hash of namespace UUID and name. func NewV5(ns UUID, name string) UUID { return global.NewV5(ns, name) } // Generator provides interface for generating UUIDs. type Generator interface { NewV1() (UUID, error) NewV2(domain byte) (UUID, error) NewV3(ns UUID, name string) UUID NewV4() (UUID, error) NewV5(ns UUID, name string) UUID } // Default generator implementation. type rfc4122Generator struct { clockSequenceOnce sync.Once hardwareAddrOnce sync.Once storageMutex sync.Mutex rand io.Reader epochFunc epochFunc hwAddrFunc hwAddrFunc lastTime uint64 clockSequence uint16 hardwareAddr [6]byte } func newRFC4122Generator() Generator { return &rfc4122Generator{ epochFunc: time.Now, hwAddrFunc: defaultHWAddrFunc, rand: rand.Reader, } } // NewV1 returns UUID based on current timestamp and MAC address. func (g *rfc4122Generator) NewV1() (UUID, error) { u := UUID{} timeNow, clockSeq, err := g.getClockSequence() if err != nil { return Nil, err } binary.BigEndian.PutUint32(u[0:], uint32(timeNow)) binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32)) binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48)) binary.BigEndian.PutUint16(u[8:], clockSeq) hardwareAddr, err := g.getHardwareAddr() if err != nil { return Nil, err } copy(u[10:], hardwareAddr) u.SetVersion(V1) u.SetVariant(VariantRFC4122) return u, nil } // NewV2 returns DCE Security UUID based on POSIX UID/GID. func (g *rfc4122Generator) NewV2(domain byte) (UUID, error) { u, err := g.NewV1() if err != nil { return Nil, err } switch domain { case DomainPerson: binary.BigEndian.PutUint32(u[:], posixUID) case DomainGroup: binary.BigEndian.PutUint32(u[:], posixGID) } u[9] = domain u.SetVersion(V2) u.SetVariant(VariantRFC4122) return u, nil } // NewV3 returns UUID based on MD5 hash of namespace UUID and name. func (g *rfc4122Generator) NewV3(ns UUID, name string) UUID { u := newFromHash(md5.New(), ns, name) u.SetVersion(V3) u.SetVariant(VariantRFC4122) return u } // NewV4 returns random generated UUID. func (g *rfc4122Generator) NewV4() (UUID, error) { u := UUID{} if _, err := io.ReadFull(g.rand, u[:]); err != nil { return Nil, err } u.SetVersion(V4) u.SetVariant(VariantRFC4122) return u, nil } // NewV5 returns UUID based on SHA-1 hash of namespace UUID and name. func (g *rfc4122Generator) NewV5(ns UUID, name string) UUID { u := newFromHash(sha1.New(), ns, name) u.SetVersion(V5) u.SetVariant(VariantRFC4122) return u } // Returns epoch and clock sequence. func (g *rfc4122Generator) getClockSequence() (uint64, uint16, error) { var err error g.clockSequenceOnce.Do(func() { buf := make([]byte, 2) if _, err = io.ReadFull(g.rand, buf); err != nil { return } g.clockSequence = binary.BigEndian.Uint16(buf) }) if err != nil { return 0, 0, err } g.storageMutex.Lock() defer g.storageMutex.Unlock() timeNow := g.getEpoch() // Clock didn't change since last UUID generation. // Should increase clock sequence. if timeNow <= g.lastTime { g.clockSequence++ } g.lastTime = timeNow return timeNow, g.clockSequence, nil } // Returns hardware address. func (g *rfc4122Generator) getHardwareAddr() ([]byte, error) { var err error g.hardwareAddrOnce.Do(func() { if hwAddr, err := g.hwAddrFunc(); err == nil { copy(g.hardwareAddr[:], hwAddr) return } // Initialize hardwareAddr randomly in case // of real network interfaces absence. if _, err = io.ReadFull(g.rand, g.hardwareAddr[:]); err != nil { return } // Set multicast bit as recommended by RFC 4122 g.hardwareAddr[0] |= 0x01 }) if err != nil { return []byte{}, err } return g.hardwareAddr[:], nil } // Returns difference in 100-nanosecond intervals between // UUID epoch (October 15, 1582) and current time. func (g *rfc4122Generator) getEpoch() uint64 { return epochStart + uint64(g.epochFunc().UnixNano()/100) } // Returns UUID based on hashing of namespace UUID and name. func newFromHash(h hash.Hash, ns UUID, name string) UUID { u := UUID{} h.Write(ns[:]) h.Write([]byte(name)) copy(u[:], h.Sum(nil)) return u } // Returns hardware address. func defaultHWAddrFunc() (net.HardwareAddr, error) { ifaces, err := net.Interfaces() if err != nil { return []byte{}, err } for _, iface := range ifaces { if len(iface.HardwareAddr) >= 6 { return iface.HardwareAddr, nil } } return []byte{}, fmt.Errorf("uuid: no HW address found") } ================================================ FILE: utils/uuid/sql.go ================================================ // Copyright (C) 2013-2018 by Maxim Bublis // // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including // without limitation the rights to use, copy, modify, merge, publish, // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: // // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. package uuid import ( "database/sql/driver" "fmt" ) // Value implements the driver.Valuer interface. func (u UUID) Value() (driver.Value, error) { return u.String(), nil } // Scan implements the sql.Scanner interface. // A 16-byte slice is handled by UnmarshalBinary, while // a longer byte slice or a string is handled by UnmarshalText. func (u *UUID) Scan(src interface{}) error { switch src := src.(type) { case []byte: if len(src) == Size { return u.UnmarshalBinary(src) } return u.UnmarshalText(src) case string: return u.UnmarshalText([]byte(src)) } return fmt.Errorf("uuid: cannot convert %T to UUID", src) } // NullUUID can be used with the standard sql package to represent a // UUID value that can be NULL in the database type NullUUID struct { UUID UUID Valid bool } // Value implements the driver.Valuer interface. func (u NullUUID) Value() (driver.Value, error) { if !u.Valid { return nil, nil } // Delegate to UUID Value function return u.UUID.Value() } // Scan implements the sql.Scanner interface. func (u *NullUUID) Scan(src interface{}) error { if src == nil { u.UUID, u.Valid = Nil, false return nil } // Delegate to UUID Scan function u.Valid = true return u.UUID.Scan(src) } ================================================ FILE: utils/uuid/uuid.go ================================================ // Copyright (C) 2013-2018 by Maxim Bublis // // Permission is hereby granted, free of charge, to any person obtaining // a copy of this software and associated documentation files (the // "Software"), to deal in the Software without restriction, including // without limitation the rights to use, copy, modify, merge, publish, // distribute, sublicense, and/or sell copies of the Software, and to // permit persons to whom the Software is furnished to do so, subject to // the following conditions: // // The above copyright notice and this permission notice shall be // included in all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND // NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE // LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION // OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // Package uuid provides implementation of Universally Unique Identifier (UUID). // Supported versions are 1, 3, 4 and 5 (as specified in RFC 4122) and // version 2 (as specified in DCE 1.1). package uuid import ( "bytes" "encoding/hex" ) // Size of a UUID in bytes. const Size = 16 // UUID representation compliant with specification // described in RFC 4122. type UUID [Size]byte // UUID versions const ( _ byte = iota V1 V2 V3 V4 V5 ) // UUID layout variants. const ( VariantNCS byte = iota VariantRFC4122 VariantMicrosoft VariantFuture ) // UUID DCE domains. const ( DomainPerson = iota DomainGroup DomainOrg ) // String parse helpers. var ( urnPrefix = []byte("urn:uuid:") byteGroups = []int{8, 4, 4, 4, 12} ) // Nil is special form of UUID that is specified to have all // 128 bits set to zero. var Nil = UUID{} // Predefined namespace UUIDs. var ( NamespaceDNS = Must(FromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8")) NamespaceURL = Must(FromString("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) NamespaceOID = Must(FromString("6ba7b812-9dad-11d1-80b4-00c04fd430c8")) NamespaceX500 = Must(FromString("6ba7b814-9dad-11d1-80b4-00c04fd430c8")) ) // Equal returns true if u1 and u2 equals, otherwise returns false. func Equal(u1 UUID, u2 UUID) bool { return bytes.Equal(u1[:], u2[:]) } // Version returns algorithm version used to generate UUID. func (u UUID) Version() byte { return u[6] >> 4 } // Variant returns UUID layout variant. func (u UUID) Variant() byte { switch { case (u[8] >> 7) == 0x00: return VariantNCS case (u[8] >> 6) == 0x02: return VariantRFC4122 case (u[8] >> 5) == 0x06: return VariantMicrosoft case (u[8] >> 5) == 0x07: fallthrough default: return VariantFuture } } // Bytes returns bytes slice representation of UUID. func (u UUID) Bytes() []byte { return u[:] } // Returns canonical string representation of UUID: // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. func (u UUID) String() string { buf := make([]byte, 36) hex.Encode(buf[0:8], u[0:4]) buf[8] = '-' hex.Encode(buf[9:13], u[4:6]) buf[13] = '-' hex.Encode(buf[14:18], u[6:8]) buf[18] = '-' hex.Encode(buf[19:23], u[8:10]) buf[23] = '-' hex.Encode(buf[24:], u[10:]) return string(buf) } // SetVersion sets version bits. func (u *UUID) SetVersion(v byte) { u[6] = (u[6] & 0x0f) | (v << 4) } // SetVariant sets variant bits. func (u *UUID) SetVariant(v byte) { switch v { case VariantNCS: u[8] = (u[8]&(0xff>>1) | (0x00 << 7)) case VariantRFC4122: u[8] = (u[8]&(0xff>>2) | (0x02 << 6)) case VariantMicrosoft: u[8] = (u[8]&(0xff>>3) | (0x06 << 5)) case VariantFuture: fallthrough default: u[8] = (u[8]&(0xff>>3) | (0x07 << 5)) } } // Must is a helper that wraps a call to a function returning (UUID, error) // and panics if the error is non-nil. It is intended for use in variable // initializations such as // var packageUUID = uuid.Must(uuid.FromString("123e4567-e89b-12d3-a456-426655440000")); func Must(u UUID, err error) UUID { if err != nil { panic(err) } return u }