Repository: mattn/qq Branch: master Commit: 7eafaf5b5152 Files: 6 Total size: 19.7 KB Directory structure: gitextract_apve2ho1/ ├── .travis.yml ├── README.md ├── cmd/ │ └── qq/ │ └── main.go ├── qq.go ├── qq_test.go └── wercker.yml ================================================ FILE CONTENTS ================================================ ================================================ FILE: .travis.yml ================================================ language: go go: - tip before_install: - go get github.com/mattn/goveralls - go get golang.org/x/tools/cmd/cover script: - $HOME/gopath/bin/goveralls -repotoken MLMW2kUwHGbt4hXxxt7uboywTNJsdqCjr ================================================ FILE: README.md ================================================ # qq [![Build Status](https://travis-ci.org/mattn/qq.svg?branch=master)](https://travis-ci.org/mattn/qq) [![Coverage Status](https://coveralls.io/repos/github/mattn/qq/badge.svg?branch=master)](https://coveralls.io/github/mattn/qq?branch=master) Select stdin with query. ## Usage ``` $ ps | qq -q "select pid from stdin" 9324 16344 13824 ``` ``` $ ps | qq -q "select command from stdin where pid = 9324" /usr/bin/grep ``` ## Requirements * go ## Installation Library install ``` $ go get github.com/mattn/qq ``` or Command install ``` $ go get github.com/mattn/qq/... ``` ## License MIT ## Author Yasuhiro Matsumoto (a.k.a. mattn) ================================================ FILE: cmd/qq/main.go ================================================ package main import ( "encoding/csv" "encoding/json" "flag" "fmt" "os" "path/filepath" "github.com/mattn/go-encoding" "github.com/mattn/qq" xenc "golang.org/x/text/encoding" ) var ( noheader = flag.Bool("nh", false, "don't treat first line as header") outheader = flag.Bool("oh", false, "output header line") inputcsv = flag.Bool("ic", false, "input csv") inputtsv = flag.Bool("it", false, "input tsv") inputltsv = flag.Bool("il", false, "input ltsv") inputpat = flag.String("ip", "", "input delimiter pattern as regexp") outputjson = flag.Bool("oj", false, "output json") outputraw = flag.Bool("or", false, "output raw") enc = flag.String("e", "", "encoding of input stream") query = flag.String("q", "", "select query") ) func main() { flag.Parse() var ee xenc.Encoding if *enc != "" { ee = encoding.GetEncoding(*enc) if ee == nil { fmt.Fprintln(os.Stderr, "invalid encoding name:", *enc) os.Exit(1) } } qq, err := qq.NewQQ(&qq.Option{ NoHeader: *noheader, OutHeader: *outheader, InputCSV: *inputcsv, InputTSV: *inputtsv, InputLTSV: *inputltsv, InputPat: *inputpat, Encoding: ee, }) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } err = qq.Import(os.Stdin, "stdin") if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } for _, fn := range flag.Args() { if fn != "-" { fb := filepath.Base(fn) f, err := os.Open(fn) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } err = qq.Import(f, fb) f.Close() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } } if *query == "" { *query = "select * from stdin" } rows, err := qq.Query(*query) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } if *outputjson { err = json.NewEncoder(os.Stdout).Encode(rows) } else if *outputraw { for _, row := range rows { for c, col := range row { if c > 0 { fmt.Print("\t") } fmt.Print(col) } fmt.Println() } } else { err = csv.NewWriter(os.Stdout).WriteAll(rows) } if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } ================================================ FILE: qq.go ================================================ package qq import ( "bufio" "database/sql" "encoding/csv" "fmt" "io" "regexp" "strings" "unicode" xenc "golang.org/x/text/encoding" "github.com/mattn/go-runewidth" _ "github.com/mattn/go-sqlite3" "github.com/najeira/ltsv" ) const ( Comma = "," ) // QQ is the most basic structure of qq type QQ struct { db *sql.DB Opt *Option } // Option is a structure that qq command can receive type Option struct { NoHeader bool OutHeader bool InputCSV bool InputTSV bool InputLTSV bool InputPat string Encoding xenc.Encoding } var renum = regexp.MustCompile(`^[+-]?[1-9][0-9]*(\.[0-9]+)?(e-?[0-9]+)?$`) func readLines(r io.Reader) ([]string, error) { var lines []string scanner := bufio.NewScanner(r) for scanner.Scan() { line := scanner.Text() if strings.TrimSpace(line) == "" { continue } lines = append(lines, line) } return lines, scanner.Err() } func (qq *QQ) lines2rows(lines []string) [][]string { cr := []rune(lines[0]) w := 0 rows := make([][]string, len(lines)) i := 0 skip_white: for ; i < len(cr); i++ { for _, line := range lines { if !unicode.IsSpace(rune(line[i])) { break skip_white } } w++ } li := i for ; i < len(cr); i++ { r := cr[i] w += runewidth.RuneWidth(r) last := i == len(cr)-1 if i == 0 || (!unicode.IsSpace(r) && !last) { continue } if last { for ri := range rows { fv := strings.TrimSpace(string(([]rune(lines[ri]))[li:])) if ri == 0 && fv == "" && !qq.Opt.NoHeader { fv = fmt.Sprintf("______f%d", ri+1) } rows[ri] = append(rows[ri], fv) } } else { part := false for _, line := range lines { pr := []rune(runewidth.Truncate(line, w, "")) if !unicode.IsSpace(pr[len(pr)-1]) { part = true break } } if !part { for ri := range rows { lr := []rune(lines[ri]) ib := i if ib >= len(lr) { ib = len(lr) - 1 } fv := strings.TrimSpace(string(lr[li:ib])) if ri == 0 && fv == "" && !qq.Opt.NoHeader { fv = fmt.Sprintf("______f%d", ri+1) } rows[ri] = append(rows[ri], fv) } for ; i < len(cr); i++ { cw := runewidth.RuneWidth(r) for _, line := range lines { pr := []rune(runewidth.Truncate(line, w+cw, "")) if !unicode.IsSpace(pr[len(pr)-1]) { part = true break } } if part { break } w += cw } li = i } } } return rows } // NewQQ creates new connection to sqlite3 func NewQQ(opt *Option) (*QQ, error) { db, err := sql.Open("sqlite3", ":memory:") if err != nil { return nil, err } return &QQ{db, opt}, nil } const ( sqliteINTEGER = "INTEGER" sqliteTEXT = "TEXT" sqliteREAL = "REAL" ) type column struct { Name string Type string } func newColumn(name string) *column { return &column{ Name: name, Type: sqliteINTEGER, } } func (qq *QQ) columnsAndRows(r io.Reader) (cn []*column, rows [][]string, err error) { if qq.Opt.Encoding != nil { r = qq.Opt.Encoding.NewDecoder().Reader(r) } if qq.Opt.InputCSV { rows, err = csv.NewReader(r).ReadAll() if err != nil { return nil, nil, err } } else if qq.Opt.InputTSV { csv := csv.NewReader(r) csv.Comma = '\t' rows, err = csv.ReadAll() if err != nil { return nil, nil, err } } else if qq.Opt.InputLTSV { rawRows, err := ltsv.NewReader(r).ReadAll() if err != nil { return nil, nil, err } keys := make(map[string]struct{}) for _, rowMap := range rawRows { for k := range rowMap { keys[k] = struct{}{} } } for k := range keys { cn = append(cn, newColumn(k)) } for _, rowMap := range rawRows { row := make([]string, len(cn)) for i, v := range cn { row[i] = rowMap[v.Name] } rows = append(rows, row) } } else if qq.Opt.InputPat != "" { lines, err := readLines(r) if err != nil { return nil, nil, err } if len(lines) == 0 { return nil, nil, nil } re, err := regexp.Compile(qq.Opt.InputPat) if err != nil { return nil, nil, err } for _, line := range lines { rows = append(rows, re.Split(line, -1)) } } else { lines, err := readLines(r) if err != nil { return nil, nil, err } if len(lines) == 0 { return nil, nil, nil } rows = qq.lines2rows(lines) } if !qq.Opt.InputLTSV { if qq.Opt.NoHeader { for i := 0; i < len(rows[0]); i++ { cn = append(cn, newColumn(fmt.Sprintf(`f%d`, i+1))) } } else { for _, v := range rows[0] { cn = append(cn, newColumn(v)) } } } if !qq.Opt.NoHeader && !qq.Opt.InputLTSV { rows = rows[1:] } for _, row := range rows { for i, col := range row { if col == "" { continue } colDef := cn[i] if colDef.Type == sqliteTEXT { continue } if matches := renum.FindStringSubmatch(col); len(matches) > 0 { if matches[1] != "" || matches[2] != "" { colDef.Type = sqliteREAL } } else { colDef.Type = sqliteTEXT } } } return cn, rows, nil } // Import from csv/tsv files or stdin func (qq *QQ) Import(r io.Reader, name string) error { cn, rows, err := qq.columnsAndRows(r) if err != nil || len(rows) == 0 { return err } s := `create table '` + strings.Replace(name, `'`, `''`, -1) + `'(` for i, n := range cn { if i > 0 { s += Comma } s += fmt.Sprintf(`'%s' %s`, strings.Replace(n.Name, `'`, `''`, -1), n.Type) } s += `)` _, err = qq.db.Exec(s) if err != nil { return err } s = `insert into '` + strings.Replace(name, `'`, `''`, -1) + `'(` for i, n := range cn { if i > 0 { s += Comma } s += `'` + strings.Replace(n.Name, `'`, `''`, -1) + `'` } s += `) values` d := `` for _, row := range rows { if d != `` { d += `,` } d += `(` for i, col := range row { if i >= len(cn) { break } if i > 0 { d += Comma } if renum.MatchString(col) { d += col } else { d += `'` + strings.Replace(col, `'`, `''`, -1) + `'` } } d += `)` } _, err = qq.db.Exec(s + d) if err != nil { return err } return nil } // Query runs a query and formatize result set func (qq *QQ) Query(query string) ([][]string, error) { qrows, err := qq.db.Query(query) if err != nil { return nil, err } defer qrows.Close() cols, err := qrows.Columns() if err != nil { return nil, err } if len(cols) == 0 { return nil, nil } rows := [][]string{} if qq.Opt.OutHeader { rows = append(rows, cols) } values := make([]interface{}, len(cols)) ptrs := make([]interface{}, len(cols)) for i := range cols { ptrs[i] = &values[i] } for qrows.Next() { err = qrows.Scan(ptrs...) if err != nil { return nil, err } cells := []string{} for _, val := range values { b, ok := val.([]byte) var v string if ok { v = string(b) } else { v = fmt.Sprint(val) } cells = append(cells, v) } rows = append(rows, cells) } return rows, nil } // Close database connection func (qq *QQ) Close() error { return qq.db.Close() } ================================================ FILE: qq_test.go ================================================ package qq import ( "io" "reflect" "strings" "testing" ) var testcasesReadlines = []struct { input string output []string }{ { input: " PID command \n" + "\n" + " 1 ls \n", output: []string{ " PID command ", " 1 ls ", }, }, } func TestReadLines(t *testing.T) { for _, testcase := range testcasesReadlines { lines, err := readLines(strings.NewReader(testcase.input)) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(lines, testcase.output) { t.Fatalf("%q should be read as %v: got %v", testcase.input, testcase.output, lines) } } } var testcasesLines2rows = []struct { input []string output [][]string }{ { input: []string{ " PID command ", " 1 ls ", }, output: [][]string{ {"PID", "command"}, {"1", "ls"}, }, }, { input: []string{ " PID command ", " 1 ls ", }, output: [][]string{ {"PID command"}, {"1 ls"}, }, }, { input: []string{ " PID command ", " 1 ls ", }, output: [][]string{ {"PID", "command"}, {"", "1 ls"}, }, }, { input: []string{ " command ", " 1 ls ", }, output: [][]string{ {"______f1", "command"}, {"1", "ls"}, }, }, { input: []string{ " 1 ", " ", }, output: [][]string{ {"1"}, {""}, }, }, { input: []string{ " ", " 1 ", }, output: [][]string{ {"______f1"}, {"1"}, }, }, { input: []string{ "a b", "1 ", }, output: [][]string{ {"a", "b"}, {"1", ""}, }, }, } func TestLines2Rows(t *testing.T) { qq, err := NewQQ(&Option{}) if err != nil { t.Fatal(err) } defer qq.Close() for _, testcase := range testcasesLines2rows { rows := qq.lines2rows(testcase.input) if !reflect.DeepEqual(rows, testcase.output) { t.Fatalf("%q should be parsed as %v: got %v", testcase.input, testcase.output, rows) } } } func test(r io.Reader, name string, query string, opt *Option) ([][]string, error) { qq, err := NewQQ(opt) if err != nil { return nil, err } defer qq.Close() err = qq.Import(r, name) if err != nil { return nil, err } rows, err := qq.Query(query) if err != nil { return nil, err } return rows, nil } func TestQQ(t *testing.T) { input := ` PID command 1 /usr/bin/ls 2 /usr/bin/grep 3 /usr/bin/php run.php --opt='1' ` rows, err := test(strings.NewReader(input), "stdin", "select pid from stdin", &Option{}) if err != nil { t.Fatal(err) } if len(rows) != 3 { t.Fatalf("rows should have three row: got %v", rows) } if len(rows[0]) != 1 { t.Fatalf("columns should have only one: got %v", rows[0]) } if rows[0][0] != "1" { t.Fatalf("first result should be 1: got %v", rows[0][0]) } if rows[1][0] != "2" { t.Fatalf("second result should be 2: got %v", rows[0][0]) } if rows[2][0] != "3" { t.Fatalf("second result should be 3: got %v", rows[0][0]) } rows, err = test(strings.NewReader(input), "stdin", "select command from stdin where pid = 2", &Option{}) if err != nil { t.Fatal(err) } if rows[0][0] != "/usr/bin/grep" { t.Fatalf("result should be '/usr/bin/grep': got %v", rows[0][0]) } rows, err = test(strings.NewReader(input), "stdin", "select command from stdin where pid = 3", &Option{}) if err != nil { t.Fatal(err) } if rows[0][0] != "/usr/bin/php run.php --opt='1'" { t.Fatalf("result should be '/usr/bin/php run.php --opt='1': got %v", rows[0][0]) } } func TestInputCSV(t *testing.T) { input := ` PID,command 1,/usr/bin/ls 2,/usr/bin/grep ` opt := &Option{ InputCSV: true, } rows, err := test(strings.NewReader(input), "stdin", "select pid from stdin", opt) if err != nil { t.Fatal(err) } if len(rows) != 2 { t.Fatalf("rows should have two row: got %v", rows) } if len(rows[0]) != 1 { t.Fatalf("columns should have only one: got %v", rows[0]) } if rows[0][0] != "1" { t.Fatalf("first result should be 1: got %v", rows[0][0]) } if rows[1][0] != "2" { t.Fatalf("second result should be 2: got %v", rows[0][0]) } rows, err = test(strings.NewReader(input), "stdin", "select command from stdin where pid = 2", opt) if err != nil { t.Fatal(err) } if rows[0][0] != "/usr/bin/grep" { t.Fatalf("result should be '/usr/bin/grep': got %v", rows[0][0]) } } func TestInputTSV(t *testing.T) { input := "PID\tcommand\n1\t/usr/bin/ls\n2\t/usr/bin/grep\n" opt := &Option{ InputTSV: true, } rows, err := test(strings.NewReader(input), "stdin", "select pid from stdin", opt) if err != nil { t.Fatal(err) } if len(rows) != 2 { t.Fatalf("rows should have two row: got %v", rows) } if len(rows[0]) != 1 { t.Fatalf("columns should have only one: got %v", rows[0]) } if rows[0][0] != "1" { t.Fatalf("first result should be 1: got %v", rows[0][0]) } if rows[1][0] != "2" { t.Fatalf("second result should be 2: got %v", rows[0][0]) } rows, err = test(strings.NewReader(input), "stdin", "select command from stdin where pid = 2", opt) if err != nil { t.Fatal(err) } if rows[0][0] != "/usr/bin/grep" { t.Fatalf("result should be '/usr/bin/grep': got %v", rows[0][0]) } } func TestInputPat(t *testing.T) { input := "PID#command\n1#/usr/bin/ls\n2#/usr/bin/grep\n" opt := &Option{ InputPat: `#`, } rows, err := test(strings.NewReader(input), "stdin", "select pid from stdin", opt) if err != nil { t.Fatal(err) } if len(rows) != 2 { t.Fatalf("rows should have two row: got %v", rows) } if len(rows[0]) != 1 { t.Fatalf("columns should have only one: got %v", rows[0]) } if rows[0][0] != "1" { t.Fatalf("first result should be 1: got %v", rows[0][0]) } if rows[1][0] != "2" { t.Fatalf("second result should be 2: got %v", rows[0][0]) } rows, err = test(strings.NewReader(input), "stdin", "select command from stdin where pid = 2", opt) if err != nil { t.Fatal(err) } if rows[0][0] != "/usr/bin/grep" { t.Fatalf("result should be '/usr/bin/grep': got %v", rows[0][0]) } } func TestNoHeader(t *testing.T) { input := ` 1 /usr/bin/ls 2 /usr/bin/grep ` opt := &Option{ NoHeader: true, } rows, err := test(strings.NewReader(input), "stdin", "select f1 from stdin", opt) if err != nil { t.Fatal(err) } if len(rows) != 2 { t.Fatalf("rows should have two row: got %v", rows) } if len(rows[0]) != 1 { t.Fatalf("columns should have only one: got %v", rows[0]) } if rows[0][0] != "1" { t.Fatalf("first result should be 1: got %v", rows[0][0]) } if rows[1][0] != "2" { t.Fatalf("second result should be 2: got %v", rows[0][0]) } rows, err = test(strings.NewReader(input), "stdin", "select f2 from stdin where f1 = 2", opt) if err != nil { t.Fatal(err) } if rows[0][0] != "/usr/bin/grep" { t.Fatalf("result should be '/usr/bin/grep': got %v", rows[0][0]) } } func TestOutHeader(t *testing.T) { input := ` PID command 1 /usr/bin/ls 2 /usr/bin/grep ` opt := &Option{ OutHeader: true, } rows, err := test(strings.NewReader(input), "stdin", "select pid from stdin", opt) if err != nil { t.Fatal(err) } if len(rows) != 3 { t.Fatalf("rows should have three row: got %v", rows) } if len(rows[0]) != 1 { t.Fatalf("columns should have only one: got %v", rows[0]) } if rows[0][0] != "PID" { t.Fatalf("first result should be 'PID': got %v", rows[0][0]) } if rows[1][0] != "1" { t.Fatalf("first result should be 1: got %v", rows[1][0]) } if rows[2][0] != "2" { t.Fatalf("second result should be 2: got %v", rows[2][0]) } rows, err = test(strings.NewReader(input), "stdin", "select command from stdin where pid = 2", opt) if err != nil { t.Fatal(err) } if rows[0][0] != "command" { t.Fatalf("result should be 'command': got %v", rows[0][0]) } if rows[1][0] != "/usr/bin/grep" { t.Fatalf("result should be '/usr/bin/grep': got %v", rows[1][0]) } } func TestInputLTSV(t *testing.T) { input := `PID:1 command:/usr/bin/ls PID:2 command:/usr/bin/grep ` opt := &Option{ InputLTSV: true, } rows, err := test(strings.NewReader(input), "stdin", "select pid from stdin", opt) if err != nil { t.Fatal(err) } if len(rows) != 2 { t.Fatalf("rows should have two row: got %v", rows) } if len(rows[0]) != 1 { t.Fatalf("columns should have only one: got %v", rows[0]) } if rows[0][0] != "1" { t.Fatalf("first result should be 1: got %v", rows[0][0]) } if rows[1][0] != "2" { t.Fatalf("second result should be 2: got %v", rows[0][0]) } rows, err = test(strings.NewReader(input), "stdin", "select command from stdin where pid = 2", opt) if err != nil { t.Fatal(err) } if rows[0][0] != "/usr/bin/grep" { t.Fatalf("result should be '/usr/bin/grep': got %v", rows[0][0]) } } func TestColumsAndRows(t *testing.T) { input := ` int_key text_key real_key 1 1 15 2 /usr/bin/grep 1.04 2 10 300065 ` qq, _ := NewQQ(&Option{}) cn, _, _ := qq.columnsAndRows(strings.NewReader(input)) out := make([]column, len(cn)) for i, c := range cn { out[i] = *c } expect := []column{ { Name: "int_key", Type: sqliteINTEGER, }, { Name: "text_key", Type: sqliteTEXT, }, { Name: "real_key", Type: sqliteREAL, }, } if !reflect.DeepEqual(out, expect) { t.Errorf("\n out =%+v\n want=%+v", out, expect) } } ================================================ FILE: wercker.yml ================================================ box: tcnksm/gox build: steps: - setup-go-workspace - script: name: show environments code: | git version go version - script: name: go get code: | go get -t ./... - script: name: go test code: | go test -v ./... deploy: steps: - setup-go-workspace - script: name: go get code: | go get ./... - wercker/gox: os: darwin linux windows arch: 386 amd64 output: '{{.Dir}}_{{.OS}}_{{.Arch}}/{{.Dir}}' dest: $WERCKER_OUTPUT_DIR/pkg - tcnksm/zip: input: $WERCKER_OUTPUT_DIR/pkg output: $WERCKER_OUTPUT_DIR/dist - script: name: set release tag code: | if [ -n "$GOBUMP_NEW_VERSION" ]; then export RELEASE_TAG="v$GOBUMP_NEW_VERSION" fi - tcnksm/ghr: token: $GITHUB_TOKEN input: $WERCKER_OUTPUT_DIR/dist replace: true version: $RELEASE_TAG opt: --draft