Repository: datasweet/datatable Branch: master Commit: 9439f126acd3 Files: 79 Total size: 196.4 KB Directory structure: gitextract_85671esv/ ├── .circleci/ │ └── config.yml ├── .editorconfig ├── .gitignore ├── .vscode/ │ └── settings.json ├── LICENSE ├── README.md ├── aggregate.go ├── aggregate_test.go ├── column.go ├── concat.go ├── concat_test.go ├── copy.go ├── copy_test.go ├── errors.go ├── eval_expr.go ├── export.go ├── export_test.go ├── go.mod ├── go.sum ├── hasher.go ├── import/ │ └── csv/ │ ├── import.go │ ├── import_test.go │ └── options.go ├── join.go ├── join_test.go ├── mutate_column.go ├── mutate_column_test.go ├── mutate_row.go ├── mutate_row_test.go ├── row.go ├── select.go ├── serie/ │ ├── converters.go │ ├── copy.go │ ├── copy_test.go │ ├── errors.go │ ├── iterate.go │ ├── iterate_test.go │ ├── mutate.go │ ├── mutate_test.go │ ├── select.go │ ├── select_test.go │ ├── serie.go │ ├── serie_bool.go │ ├── serie_bool_test.go │ ├── serie_float32.go │ ├── serie_float32_test.go │ ├── serie_float64.go │ ├── serie_float64_test.go │ ├── serie_int.go │ ├── serie_int32.go │ ├── serie_int32_test.go │ ├── serie_int64.go │ ├── serie_int64_test.go │ ├── serie_int_test.go │ ├── serie_raw.go │ ├── serie_raw_test.go │ ├── serie_string.go │ ├── serie_string_test.go │ ├── serie_test.go │ ├── serie_time.go │ ├── serie_time_test.go │ ├── sort.go │ ├── sort_test.go │ ├── stat.go │ ├── stat_test.go │ └── utils_test.go ├── serie_test.go ├── sort.go ├── sort_test.go ├── spec.md ├── table.go ├── table_print.go ├── table_print_test.go ├── table_test.go ├── test/ │ ├── main.go │ └── phone_data.csv ├── utils_test.go ├── where.go └── where_test.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .circleci/config.yml ================================================ version: 2 jobs: build: docker: - image: circleci/golang:1.14 steps: - checkout - run: name: Tests command: | go fmt ./... go vet ./... go test -v ./... ================================================ FILE: .editorconfig ================================================ # http://editorconfig.org root = true [*] indent_size = 2 indent_style = space charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true [*.go] indent_style = tab [*.{js,jsx,json,html}] indent_size = 4 [Makefile] indent_style = tab [*.md] trim_trailing_whitespace = false ================================================ FILE: .gitignore ================================================ vendor/ bin/ data/ .DS_Store ================================================ FILE: .vscode/settings.json ================================================ { "go.testFlags": ["-v", "-count=1"] } ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2017-2020 Datasweet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # datatable [![Go Report Card](https://goreportcard.com/badge/github.com/datasweet/datatable)](https://goreportcard.com/report/github.com/datasweet/datatable) [![GoDoc](https://godoc.org/github.com/datasweet/datatable?status.png)](https://godoc.org/github.com/datasweet/datatable) [![GitHub stars](https://img.shields.io/github/stars/datasweet/datatable.svg)](https://github.com/datasweet/datatable/stargazers) [![GitHub license](https://img.shields.io/github/license/datasweet/datatable.svg)](https://github.com/datasweet/datatable/blob/master/LICENSE) [![datasweet-logo](https://www.datasweet.fr/wp-content/uploads/2019/02/datasweet-black.png)](http://www.datasweet.fr) datatable is a Go package to manipulate tabular data, like an excel spreadsheet. datatable is inspired by the pandas python package and the data.frame R structure. Although it's production ready, be aware that we're still working on API improvements ## Installation ``` go get github.com/datasweet/datatable ``` ## Features - Create custom Series (ie custom columns). Currently available, serie.Int, serie.String, serie.Time, serie.Float64. - Apply expressions - Selects (head, tail, subset) - Sorting - InnerJoin, LeftJoin, RightJoin, OuterJoin, Concats - Aggregate - Import from CSV - Export to map, slice ### Creating a DataTable ```go package main import ( "fmt" "github.com/datasweet/datatable" ) func main() { dt := datatable.New("test") dt.AddColumn("champ", datatable.String, datatable.Values("Malzahar", "Xerath", "Teemo")) dt.AddColumn("champion", datatable.String, datatable.Expr("upper(`champ`)")) dt.AddColumn("win", datatable.Int, datatable.Values(10, 20, 666)) dt.AddColumn("loose", datatable.Int, datatable.Values(6, 5, 666)) dt.AddColumn("winRate", datatable.Float64, datatable.Expr("`win` * 100 / (`win` + `loose`)")) dt.AddColumn("winRate %", datatable.String, datatable.Expr(" `winRate` ~ \" %\"")) dt.AddColumn("sum", datatable.Float64, datatable.Expr("sum(`win`)")) fmt.Println(dt) } /* CHAMP CHAMPION WIN LOOSE WINRATE WINRATE % SUM Malzahar MALZAHAR 10 6 62.5 62.5 % 696 Xerath XERATH 20 5 80 80 % 696 Teemo TEEMO 666 666 50 50 % 696 */ ``` ### Reading a CSV and aggregate ```go package main import ( "fmt" "log" "os" "time" "github.com/datasweet/datatable" "github.com/datasweet/datatable/import/csv" ) func main() { dt, err := csv.Import("csv", "phone_data.csv", csv.HasHeader(true), csv.AcceptDate("02/01/06 15:04"), csv.AcceptDate("2006-01"), ) if err != nil { log.Fatalf("reading csv: %v", err) } dt.Print(os.Stdout, datatable.PrintMaxRows(24)) dt2, err := dt.Aggregate(datatable.AggregateBy{Type: datatable.Count, Field: "index"}) if err != nil { log.Fatalf("aggregate COUNT('index'): %v", err) } fmt.Println(dt2) groups, err := dt.GroupBy(datatable.GroupBy{ Name: "year", Type: datatable.Int, Keyer: func(row datatable.Row) (interface{}, bool) { if d, ok := row["date"]; ok { if tm, ok := d.(time.Time); ok { return tm.Year(), true } } return nil, false }, }) if err != nil { log.Fatalf("GROUP BY 'year': %v", err) } dt3, err := groups.Aggregate( datatable.AggregateBy{Type: datatable.Sum, Field: "duration"}, datatable.AggregateBy{Type: datatable.CountDistinct, Field: "network"}, ) if err != nil { log.Fatalf("Aggregate SUM('duration'), COUNT_DISTINCT('network') GROUP BY 'year': %v", err) } fmt.Println(dt3) } ``` ### Creating a custom serie To create a custom serie you must provide: - a caster function, to cast a generic value to your serie value. The signature must be func(i interface{}) T - a comparator, to compare your serie value. The signature must be func(a, b T) int Example with a NullInt ```go // IntN is an alis to create the custom Serie to manage IntN func IntN(v ...interface{}) Serie { s, _ := New(NullInt{}, asNullInt, compareNullInt) if len(v) > 0 { s.Append(v...) } return s } type NullInt struct { Int int Valid bool } // Interface() to render the current struct as a value. // If not provided, the serie.All() or serie.Get() wills returns the embedded value // IE: NullInt{} func (i NullInt) Interface() interface{} { if i.Valid { return i.Int } return nil } // asNullInt is our caster function func asNullInt(i interface{}) NullInt { var ni NullInt if i == nil { return ni } if v, ok := i.(NullInt); ok { return v } if v, err := cast.ToIntE(i); err == nil { ni.Int = v ni.Valid = true } return ni } // compareNullInt is our comparator function // used to sort func compareNullInt(a, b NullInt) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } if a.Int == b.Int { return Eq } if a.Int < b.Int { return Lt } return Gt } ``` ## Who are we ? We are Datasweet, a french startup providing full service (big) data solutions. ## Questions ? problems ? suggestions ? If you find a bug or want to request a feature, please create a [GitHub Issue](https://github.com/datasweet/datatable/issues/new). ## Contributors

Cléo Rebert
## License ``` This software is licensed under the Apache License, version 2 ("ALv2"), quoted below. Copyright 2017-2020 Datasweet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ``` ================================================ FILE: aggregate.go ================================================ package datatable import ( "bytes" "encoding/gob" "fmt" "github.com/cespare/xxhash" "github.com/datasweet/datatable/serie" "github.com/pkg/errors" ) // GroupBy defines the group by configuration // Name is the name of the output column // Type is the type of the output column // Keyer is our main function to aggregate type GroupBy struct { Name string Type ColumnType Keyer func(row Row) (interface{}, bool) } // AggregationType defines the avalaible aggregation type AggregationType uint8 const ( Avg AggregationType = iota Count CountDistinct Cusum Max Min Median Stddev Sum Variance ) func (a AggregationType) String() string { switch a { case Avg: return "avg" case Count: return "count" case CountDistinct: return "count_distinct" case Cusum: return "cusum" case Max: return "max" case Min: return "min" case Median: return "median" case Stddev: return "stddev" case Sum: return "sum" case Variance: return "variance" default: panic("unkwown aggregation type") } } // AggregateBy defines the aggregation type AggregateBy struct { Type AggregationType Field string As string } // GroupBy splits our datatable by group func (dt *DataTable) GroupBy(by ...GroupBy) (*Groups, error) { if len(by) == 0 { return nil, ErrNoGroupBy } var groups []*group gindex := make(map[uint64]int) for pos := 0; pos < dt.nrows; pos++ { row := dt.Row(pos) buf := bytes.NewBuffer(nil) enc := gob.NewEncoder(buf) buckets := make([]interface{}, len(by)) for i, k := range by { k := &k if v, ok := k.Keyer(row); ok { buckets[i] = v enc.Encode(v) } } hash := xxhash.Sum64(buf.Bytes()) if at, ok := gindex[hash]; ok { groups[at].Rows = append(groups[at].Rows, pos) } else { gindex[hash] = len(groups) groups = append(groups, &group{ Key: hash, Buckets: buckets, Rows: []int{pos}, }) } } return &Groups{dt: dt, groups: groups, by: by}, nil } // Aggregate aggregates some field func (dt *DataTable) Aggregate(by ...AggregateBy) (*DataTable, error) { g := &Groups{ dt: dt, groups: []*group{ &group{TakeAll: true}, }, } return g.Aggregate(by...) } // Groups type Groups struct { dt *DataTable by []GroupBy groups []*group } type group struct { Key uint64 Buckets []interface{} Rows []int TakeAll bool } // Aggregate our groups func (g *Groups) Aggregate(aggs ...AggregateBy) (*DataTable, error) { if g == nil { return nil, ErrNoGroups } if g.dt == nil { return nil, ErrNilDatatable } // check cols series := make(map[string]serie.Serie) for _, agg := range aggs { col := g.dt.Column(agg.Field) if col == nil { err := errors.Errorf("column '%s' not found", agg.Field) return nil, errors.Wrap(err, ErrColumnNotFound.Error()) } switch agg.Type { case Avg, Count, CountDistinct, Cusum, Max, Min, Median, Stddev, Sum, Variance: series[agg.Field] = col.(*column).serie default: return nil, ErrUnknownAgg } } out := New(g.dt.name) // create columns for _, by := range g.by { typ := by.Type if len(typ) == 0 { typ = Raw } if err := out.AddColumn(by.Name, typ); err != nil { err = errors.Wrapf(err, "can't add column '%s'", by.Name) return nil, errors.Wrap(err, ErrCantAddColumn.Error()) } } for _, agg := range aggs { name := agg.As if len(name) == 0 { name = fmt.Sprintf("%s %s", agg.Type, agg.Field) } typ := Float64 switch agg.Type { case Count, CountDistinct: typ = Int64 default: } if err := out.AddColumn(name, typ); err != nil { err = errors.Wrapf(err, "can't add column '%s'", name) return nil, errors.Wrap(err, ErrCantAddColumn.Error()) } } // aggregate the series for _, group := range g.groups { values := make([]interface{}, 0, len(group.Buckets)+len(aggs)) values = append(values, group.Buckets...) for _, agg := range aggs { serie := series[agg.Field] if !group.TakeAll { serie = serie.Pick(group.Rows...) } switch agg.Type { case Avg: values = append(values, serie.Avg()) case Count: values = append(values, serie.Count()) case CountDistinct: values = append(values, serie.CountDistinct()) case Cusum: values = append(values, serie.Cusum()) case Max: values = append(values, serie.Max()) case Min: values = append(values, serie.Min()) case Median: values = append(values, serie.Median()) case Stddev: values = append(values, serie.Stddev()) case Sum: values = append(values, serie.Sum()) case Variance: values = append(values, serie.Variance()) } } out.AppendRow(values...) } return out, nil } ================================================ FILE: aggregate_test.go ================================================ package datatable_test import ( "fmt" "testing" "time" "github.com/datasweet/datatable" "github.com/stretchr/testify/assert" ) func TestAggregate(t *testing.T) { customers, orders := sampleForJoin() dt, err := customers.LeftJoin(orders, datatable.On("[Customers].[id]", "[Orders].[user_id]")) assert.NoError(t, err) assert.NotNil(t, dt) fmt.Println(dt) // Aggregate by SUM out, err := dt.Aggregate(datatable.AggregateBy{datatable.Sum, "prix_total", "sum_prix_total"}) assert.NoError(t, err) assert.NotNil(t, out) fmt.Println(out) // Aggregate by SUM(prix_total), COUNT_DISTINCT(ville) out, err = dt.Aggregate(datatable.AggregateBy{datatable.Sum, "prix_total", "sum_prix_total"}, datatable.AggregateBy{datatable.CountDistinct, "ville", "uniq_count_ville"}) assert.NoError(t, err) assert.NotNil(t, out) fmt.Println(out) groups, err := dt.GroupBy( datatable.GroupBy{ Name: "Year", Type: datatable.Int64, Keyer: func(row datatable.Row) (interface{}, bool) { t, ok := row["date_achat"].(time.Time) if !ok { return 0, false } return t.Year(), true }, }, datatable.GroupBy{ Name: "Month", Type: datatable.Int, Keyer: func(row datatable.Row) (interface{}, bool) { t, ok := row["date_achat"].(time.Time) if !ok { return 0, false } return int(t.Month()), true }, }, ) assert.NoError(t, err) assert.NotNil(t, groups) gdt, err := groups.Aggregate(datatable.AggregateBy{datatable.Sum, "prix_total", "sum_prix_total"}) assert.NoError(t, err) fmt.Println(gdt) } ================================================ FILE: column.go ================================================ package datatable import ( "reflect" "strings" "github.com/datasweet/datatable/serie" "github.com/datasweet/expr" "github.com/pkg/errors" ) // ColumnType defines the valid column type in datatable type ColumnType string const ( Bool ColumnType = "bool" String ColumnType = "string" Int ColumnType = "int" // Int8 ColumnType = "int8" // Int16 ColumnType = "int16" Int32 ColumnType = "int32" Int64 ColumnType = "int64" // Uint ColumnType = "uint" // Uint8 ColumnType = "uint8" // Uint16 ColumnType = "uint16" // Uint32 ColumnType = "uint32" // Uint64 ColumnType = "uint64" Float32 ColumnType = "float32" Float64 ColumnType = "float64" Time ColumnType = "time" Raw ColumnType = "raw" ) // ColumnOptions describes options to be apply on a column type ColumnOptions struct { Hidden bool Expr string Values []interface{} TimeFormats []string } // ColumnOption sets column options type ColumnOption func(opts *ColumnOptions) // ColumnHidden sets the visibility func ColumnHidden(v bool) ColumnOption { return func(opts *ColumnOptions) { opts.Hidden = v } } // Expr sets the expr for the column // Incompatible with ColumnValues func Expr(v string) ColumnOption { return func(opts *ColumnOptions) { opts.Expr = v } } // Values fills the column with the values // Incompatible with ColumnExpr func Values(v ...interface{}) ColumnOption { return func(opts *ColumnOptions) { opts.Values = v } } // TimeFormats sets the valid time formats. // Only for Time Column func TimeFormats(v ...string) ColumnOption { return func(opts *ColumnOptions) { opts.TimeFormats = append(opts.TimeFormats, v...) } } // ColumnSerier to create a serie from column options type ColumnSerier func(ColumnOptions) serie.Serie // ctypes is our column type registry var ctypes map[ColumnType]ColumnSerier func init() { ctypes = make(map[ColumnType]ColumnSerier) RegisterColumnType(Bool, func(opts ColumnOptions) serie.Serie { return serie.BoolN(opts.Values...) }) RegisterColumnType(String, func(opts ColumnOptions) serie.Serie { return serie.StringN(opts.Values...) }) RegisterColumnType(Int, func(opts ColumnOptions) serie.Serie { return serie.IntN(opts.Values...) }) RegisterColumnType(Int32, func(opts ColumnOptions) serie.Serie { return serie.Int32N(opts.Values...) }) RegisterColumnType(Int64, func(opts ColumnOptions) serie.Serie { return serie.Int64N(opts.Values...) }) RegisterColumnType(Float32, func(opts ColumnOptions) serie.Serie { return serie.Float32N(opts.Values...) }) RegisterColumnType(Float64, func(opts ColumnOptions) serie.Serie { return serie.Float64N(opts.Values...) }) RegisterColumnType(Time, func(opts ColumnOptions) serie.Serie { sr := serie.TimeN(opts.TimeFormats...) if len(opts.Values) > 0 { sr.Append(opts.Values...) } return sr }) RegisterColumnType(Raw, func(opts ColumnOptions) serie.Serie { return serie.Raw(opts.Values...) }) } // RegisterColumnType to extends the known type func RegisterColumnType(name ColumnType, serier ColumnSerier) error { name = ColumnType(strings.TrimSpace(string(name))) if len(name) == 0 { return ErrEmptyName } if serier == nil { return ErrNilFactory } if _, ok := ctypes[name]; ok { err := errors.Errorf("type '%s' already exists", name) return errors.Wrap(err, ErrTypeAlreadyExists.Error()) } ctypes[name] = serier return nil } // ColumnTypes to list all column type func ColumnTypes() []ColumnType { ctyp := make([]ColumnType, 0, len(ctypes)) for k := range ctypes { ctyp = append(ctyp, k) } return ctyp } // newColumnSerie to create a serie from a known type func newColumnSerie(ctyp ColumnType, options ColumnOptions) (serie.Serie, error) { if s, ok := ctypes[ctyp]; ok { return s(options), nil } err := errors.Errorf("unknown column type '%s'", ctyp) return nil, errors.Wrap(err, ErrUnknownColumnType.Error()) } // Column describes a column in our datatable type Column interface { Name() string Type() ColumnType UnderlyingType() reflect.Type IsVisible() bool IsComputed() bool //Clone(includeValues bool) Column } type column struct { name string typ ColumnType hidden bool formulae string expr expr.Node serie serie.Serie } func (c *column) Name() string { return c.name } func (c *column) Type() ColumnType { return c.typ } func (c *column) UnderlyingType() reflect.Type { return c.serie.Type() } func (c *column) IsVisible() bool { return !c.hidden } func (c *column) IsComputed() bool { return len(c.formulae) > 0 } func (c *column) emptyCopy() *column { cpy := &column{ name: c.name, typ: c.typ, hidden: c.hidden, formulae: c.formulae, serie: c.serie.EmptyCopy(), } if len(cpy.formulae) > 0 { if parsed, err := expr.Parse(cpy.formulae); err == nil { cpy.expr = parsed } } return cpy } func (c *column) copy() *column { cpy := &column{ name: c.name, typ: c.typ, hidden: c.hidden, formulae: c.formulae, serie: c.serie.Copy(), } if len(cpy.formulae) > 0 { if parsed, err := expr.Parse(cpy.formulae); err == nil { cpy.expr = parsed } } return cpy } ================================================ FILE: concat.go ================================================ package datatable // Concat datatables func (left *DataTable) Concat(table ...*DataTable) (*DataTable, error) { out := left.EmptyCopy() out.dirty = true tables := make([]*DataTable, 0, 1+len(table)) tables = append(tables, left) tables = append(tables, table...) for _, t := range tables { if t == nil { continue } for _, tc := range t.cols { pos := out.ColumnIndex(tc.name) if pos >= 0 { oc := out.cols[pos] if oc.IsComputed() { oc.serie.Grow(out.nrows - oc.serie.Len() + tc.serie.Len()) continue } if err := oc.serie.Concat(tc.serie); err != nil { return nil, err } } else { out.cols = append(out.cols, tc.emptyCopy()) oc := out.cols[len(out.cols)-1] oc.serie.Grow(out.nrows - oc.serie.Len()) if oc.IsComputed() { oc.serie.Grow(tc.serie.Len()) continue } if err := oc.serie.Concat(tc.serie); err != nil { return nil, err } } } out.nrows += t.nrows } // check for _, oc := range out.cols { size := out.nrows - oc.serie.Len() if size > 0 { oc.serie.Grow(size) } } return out, nil } // Concat datatables func Concat(tables []*DataTable) (*DataTable, error) { switch len(tables) { case 0: return nil, ErrNoTables case 1: return tables[0].Concat() default: return tables[0].Concat(tables[1:]...) } } ================================================ FILE: concat_test.go ================================================ package datatable_test import ( "testing" "time" "github.com/datasweet/datatable" "github.com/stretchr/testify/assert" ) // Sample from https://sql.sh/cours/union func sampleForConcat(t *testing.T) (*datatable.DataTable, *datatable.DataTable, *datatable.DataTable) { a := datatable.New("magasin1") a.AddColumn("prenom", datatable.String) a.AddColumn("nom", datatable.String) a.AddColumn("ville", datatable.String) a.AddColumn("date_naissance", datatable.Time) a.AddColumn("total_achat", datatable.Int64) a.AppendRow("Léon", "Dupuis", "Paris", "1983-03-06", 135) a.AppendRow("Marie", "Bernard", "Paris", "1993-07-03", 75) a.AppendRow("Sophie", "Dupond", "Marseille", "1986-02-22", 27) a.AppendRow("Marcel", "Martin", "Paris", "1976-11-24", 39) b := datatable.New("magasin2") b.AddColumn("prenom", datatable.String) b.AddColumn("nom", datatable.String) b.AddColumn("ville", datatable.String) b.AddColumn("date_naissance", datatable.Time) b.AddColumn("total_achat", datatable.Int64) b.AppendRow("Marion", "Leroy", "Lyon", "1982-10-27", 285) b.AppendRow("Paul", "Moreau", "Lyon", "1976-04-19", 133) b.AppendRow("Marie", "Bernard", "Paris", "1993-07-03", 75) b.AppendRow("Marcel", "Martin", "Paris", "1976-11-24", 39) c := datatable.New("magasin3") c.AddColumn("prenom", datatable.String) c.AddColumn("nom", datatable.String) c.AddColumn("ville", datatable.String) c.AddColumn("date_naissance", datatable.Time) c.AddColumn("marge", datatable.Float64) c.AppendRow("Marion", "Leroy", "Lyon", "1982-10-27", 5.2) c.AppendRow("Marie", "Bernard", "Paris", "1993-07-03", 0.8) return a, b, c } func TestSimpleConcat(t *testing.T) { a, b, _ := sampleForConcat(t) dt, err := a.Concat(b) assert.NoError(t, err) assert.Equal(t, "magasin1", dt.Name()) assert.Equal(t, 8, dt.NumRows()) checkTable(t, dt, "prenom", "nom", "ville", "date_naissance", "total_achat", "Léon", "Dupuis", "Paris", time.Date(1983, time.March, 6, 0, 0, 0, 0, time.UTC), int64(135), "Marie", "Bernard", "Paris", time.Date(1993, time.July, 3, 0, 0, 0, 0, time.UTC), int64(75), "Sophie", "Dupond", "Marseille", time.Date(1986, time.February, 22, 0, 0, 0, 0, time.UTC), int64(27), "Marcel", "Martin", "Paris", time.Date(1976, time.November, 24, 0, 0, 0, 0, time.UTC), int64(39), "Marion", "Leroy", "Lyon", time.Date(1982, time.October, 27, 0, 0, 0, 0, time.UTC), int64(285), "Paul", "Moreau", "Lyon", time.Date(1976, time.April, 19, 0, 0, 0, 0, time.UTC), int64(133), "Marie", "Bernard", "Paris", time.Date(1993, time.July, 3, 0, 0, 0, 0, time.UTC), int64(75), "Marcel", "Martin", "Paris", time.Date(1976, time.November, 24, 0, 0, 0, 0, time.UTC), int64(39), ) } func TestGrowColConcat(t *testing.T) { a, b, c := sampleForConcat(t) dt, err := a.Concat(b, c) assert.NoError(t, err) assert.Equal(t, "magasin1", dt.Name()) assert.Equal(t, 10, dt.NumRows()) checkTable(t, dt, "prenom", "nom", "ville", "date_naissance", "total_achat", "marge", "Léon", "Dupuis", "Paris", time.Date(1983, time.March, 6, 0, 0, 0, 0, time.UTC), int64(135), nil, "Marie", "Bernard", "Paris", time.Date(1993, time.July, 3, 0, 0, 0, 0, time.UTC), int64(75), nil, "Sophie", "Dupond", "Marseille", time.Date(1986, time.February, 22, 0, 0, 0, 0, time.UTC), int64(27), nil, "Marcel", "Martin", "Paris", time.Date(1976, time.November, 24, 0, 0, 0, 0, time.UTC), int64(39), nil, "Marion", "Leroy", "Lyon", time.Date(1982, time.October, 27, 0, 0, 0, 0, time.UTC), int64(285), nil, "Paul", "Moreau", "Lyon", time.Date(1976, time.April, 19, 0, 0, 0, 0, time.UTC), int64(133), nil, "Marie", "Bernard", "Paris", time.Date(1993, time.July, 3, 0, 0, 0, 0, time.UTC), int64(75), nil, "Marcel", "Martin", "Paris", time.Date(1976, time.November, 24, 0, 0, 0, 0, time.UTC), int64(39), nil, "Marion", "Leroy", "Lyon", time.Date(1982, time.October, 27, 0, 0, 0, 0, time.UTC), nil, float64(5.2), "Marie", "Bernard", "Paris", time.Date(1993, time.July, 3, 0, 0, 0, 0, time.UTC), nil, float64(0.8), ) } func TestConcatWithExpr(t *testing.T) { a, b, _ := sampleForConcat(t) a.AddColumn("upper_ville", datatable.String, datatable.Expr("UPPER(ville)")) b.AddColumn("upper_ville", datatable.String, datatable.Expr("UPPER(ville)")) dt, err := a.Concat(b) assert.NoError(t, err) assert.Equal(t, "magasin1", dt.Name()) assert.Equal(t, 8, dt.NumRows()) checkTable(t, dt, "prenom", "nom", "ville", "date_naissance", "total_achat", "upper_ville", "Léon", "Dupuis", "Paris", time.Date(1983, time.March, 6, 0, 0, 0, 0, time.UTC), int64(135), "PARIS", "Marie", "Bernard", "Paris", time.Date(1993, time.July, 3, 0, 0, 0, 0, time.UTC), int64(75), "PARIS", "Sophie", "Dupond", "Marseille", time.Date(1986, time.February, 22, 0, 0, 0, 0, time.UTC), int64(27), "MARSEILLE", "Marcel", "Martin", "Paris", time.Date(1976, time.November, 24, 0, 0, 0, 0, time.UTC), int64(39), "PARIS", "Marion", "Leroy", "Lyon", time.Date(1982, time.October, 27, 0, 0, 0, 0, time.UTC), int64(285), "LYON", "Paul", "Moreau", "Lyon", time.Date(1976, time.April, 19, 0, 0, 0, 0, time.UTC), int64(133), "LYON", "Marie", "Bernard", "Paris", time.Date(1993, time.July, 3, 0, 0, 0, 0, time.UTC), int64(75), "PARIS", "Marcel", "Martin", "Paris", time.Date(1976, time.November, 24, 0, 0, 0, 0, time.UTC), int64(39), "PARIS", ) } ================================================ FILE: copy.go ================================================ package datatable // EmptyCopy copies the structure of datatable (no values) func (t *DataTable) EmptyCopy() *DataTable { cpy := &DataTable{ name: t.name, dirty: t.dirty, hasExpr: t.hasExpr, nrows: 0, cols: make([]*column, len(t.cols)), } for i, col := range t.cols { cpy.cols[i] = col.emptyCopy() } return cpy } // Copy the datatable func (t *DataTable) Copy() *DataTable { cpy := &DataTable{ name: t.name, dirty: t.dirty, hasExpr: t.hasExpr, nrows: t.nrows, cols: make([]*column, len(t.cols)), } for i, col := range t.cols { cpy.cols[i] = col.copy() } return cpy } ================================================ FILE: copy_test.go ================================================ package datatable_test import ( "testing" "github.com/stretchr/testify/assert" ) func TestEmptyCopy(t *testing.T) { tb := New(t) cpy := tb.EmptyCopy() assert.NotNil(t, cpy) assert.NotSame(t, tb, cpy) assert.Equal(t, 0, cpy.NumRows()) assert.Equal(t, tb.NumCols(), cpy.NumCols()) } func TestCopy(t *testing.T) { tb := New(t) cpy := tb.Copy() assert.NotNil(t, cpy) assert.NotSame(t, tb, cpy) assert.Equal(t, tb.NumRows(), cpy.NumRows()) assert.Equal(t, tb.NumCols(), cpy.NumCols()) checkTable(t, cpy, "champ", "champion", "win", "loose", "winRate", "sum", "ok", "Malzahar", "MALZAHAR", 10, 6, "62.5 %", 696.0, true, "Xerath", "XERATH", 20, 5, "80 %", 696.0, true, "Teemo", "TEEMO", 666, 666, "50 %", 696.0, true, ) } ================================================ FILE: errors.go ================================================ package datatable import ( "github.com/pkg/errors" ) // Errors in import/csv var ( ErrOpenFile = errors.New("open file") ErrCantReadHeaders = errors.New("can't read headers") ErrReadingLine = errors.New("could not read line") ErrNilDatas = errors.New("nil datas") ErrWrongNumberOfTypes = errors.New("expected different number of types") ErrAddingColumn = errors.New("could not add column with given type") ) // Errors in aggregate.go var ( ErrNoGroupBy = errors.New("no groupby") ErrNoGroups = errors.New("no groups") ErrNilDatatable = errors.New("nil datatable") ErrColumnNotFound = errors.New("column not found") ErrUnknownAgg = errors.New("unknown agg") ErrCantAddColumn = errors.New("can't add column") ) // Errors in column.go var ( ErrEmptyName = errors.New("empty name") ErrNilFactory = errors.New("nil factory") ErrTypeAlreadyExists = errors.New("type already exists") ErrUnknownColumnType = errors.New("unknown column type") ) // Errors in concat.go var ( ErrNoTables = errors.New("no tables") ) // Errors in eval_expr var ( ErrEvaluateExprSizeMismatch = errors.New("size mismatch") ) // Errors in join.go var ( ErrNilOutputDatatable = errors.New("nil output datatable") ErrNoOutput = errors.New("no output") ErrNilTable = errors.New("table is nil") ErrNotEnoughDatatables = errors.New("not enough datatables") ErrNoOnClauses = errors.New("no on clauses") ErrOnClauseIsNil = errors.New("on clause is nil") ErrUnknownMode = errors.New("unknown mode") ) // Errors in mutate_column.go var ( ErrNilColumn = errors.New("nil column") ErrNilColumnName = errors.New("nil column name") ErrNilColumnType = errors.New("nil column type") ErrColumnAlreadyExists = errors.New("column already exists") ErrFormulaeSyntax = errors.New("formulae syntax") ErrNilSerie = errors.New("nil serie") ErrCreateSerie = errors.New("create serie") ) // Errors in mutate_rows.go var ( ErrLengthMismatch = errors.New("length mismatch") ErrUpdateRow = errors.New("update row") ) ================================================ FILE: eval_expr.go ================================================ package datatable import "github.com/pkg/errors" // evaluateExpressions to evaluate all columns with a binded expression func (t *DataTable) evaluateExpressions() error { if !t.dirty || !t.hasExpr { return nil } var cols []int var exprCols []int for i, c := range t.cols { if c.IsComputed() { exprCols = append(exprCols, i) } else { cols = append(cols, i) } } l := len(exprCols) if l == 0 { t.dirty = false return nil } // Initialize params params := make(map[string][]interface{}, len(t.cols)) for _, pos := range cols { col := t.cols[pos] params[col.name] = col.serie.All() } // Evaluate for _, idx := range exprCols { col := t.cols[idx] res, err := col.expr.Eval(params) if err != nil { return err } name := col.Name() if arr, ok := res.([]interface{}); ok { // Is array ls := col.serie.Len() la := len(arr) if t.nrows != ls || la != ls { err := errors.Errorf("evaluate expr : size mismatch %d vs %d", la, ls) return errors.Wrap(err, ErrEvaluateExprSizeMismatch.Error()) } for i := 0; i < t.nrows; i++ { col.serie.Set(i, arr[i]) } } else { // Is scalar for i := 0; i < t.nrows; i++ { col.serie.Set(i, res) } } // update dependency params[name] = col.serie.All() } t.dirty = false return nil } ================================================ FILE: export.go ================================================ package datatable // ExportOptions to add options for exporting (like showing hidden columns) type ExportOptions struct { WithHiddenCols bool } type ExportOption func(*ExportOptions) // ExportHidden to show a column when exporting (default false) func ExportHidden(v bool) ExportOption { return func(opts *ExportOptions) { opts.WithHiddenCols = v } } // newExportOptions to build the ExportOptions in order to acces the parameters func newExportOptions(opt ...ExportOption) ExportOptions { var opts ExportOptions for _, o := range opt { o(&opts) } return opts } // ToMap to export the datatable to a json-like struct func (t *DataTable) ToMap(opt ...ExportOption) []map[string]interface{} { if t == nil { return nil } opts := newExportOptions(opt...) if err := t.evaluateExpressions(); err != nil { panic(err) } // visible columns cols := make(map[string]int) for i, col := range t.cols { if opts.WithHiddenCols || col.IsVisible() { cols[col.Name()] = i } } rows := make([]map[string]interface{}, 0, t.nrows) for i := 0; i < t.nrows; i++ { r := make(map[string]interface{}, len(cols)) for name, pos := range cols { r[name] = t.cols[pos].serie.Get(i) } rows = append(rows, r) } return rows } // ToTable to export the datatable to a csv-like struct func (t *DataTable) ToTable(opt ...ExportOption) [][]interface{} { if t == nil { return nil } opts := newExportOptions(opt...) if err := t.evaluateExpressions(); err != nil { panic(err) } rows := make([][]interface{}, 0, t.nrows+1) // visible columns var headers []interface{} var cols []int for i, col := range t.cols { if opts.WithHiddenCols || col.IsVisible() { cols = append(cols, i) headers = append(headers, col.Name()) } } rows = append(rows, headers) for i := 0; i < t.nrows; i++ { r := make([]interface{}, 0, len(cols)) for _, pos := range cols { r = append(r, t.cols[pos].serie.Get(i)) } rows = append(rows, r) } return rows } // Schema describes a datatable type Schema struct { Name string `json:"name"` Columns []SchemaColumn `json:"cols"` Rows [][]interface{} `json:"rows"` } type SchemaColumn struct { Name string `json:"name"` Type string `json:"type"` } // ToSchema to export the datatable to a schema struct func (t *DataTable) ToSchema(opt ...ExportOption) *Schema { if t == nil { return nil } opts := newExportOptions(opt...) if err := t.evaluateExpressions(); err != nil { panic(err) } schema := &Schema{ Name: t.name, Rows: make([][]interface{}, 0, t.nrows), } // visible columns var cols []int for i, col := range t.cols { if opts.WithHiddenCols || col.IsVisible() { cols = append(cols, i) schema.Columns = append(schema.Columns, SchemaColumn{Type: col.UnderlyingType().Name(), Name: col.Name()}) } } for i := 0; i < t.nrows; i++ { r := make([]interface{}, 0, len(cols)) for _, pos := range cols { r = append(r, t.cols[pos].serie.Get(i)) } schema.Rows = append(schema.Rows, r) } return schema } ================================================ FILE: export_test.go ================================================ package datatable_test import ( "encoding/json" "testing" "github.com/datasweet/datatable" "github.com/stretchr/testify/assert" ) func sampleForExport(t *testing.T) *datatable.DataTable { customers := datatable.New("Customers") err := customers.AddColumn("id", datatable.Int) assert.NoError(t, err) err = customers.AddColumn("prenom", datatable.String) assert.NoError(t, err) err = customers.AddColumn("nom", datatable.String) assert.NoError(t, err) //dc.Hidden(true) err = customers.AddColumn("expr_nom", datatable.String, datatable.Expr("`prenom` ~ ' ' ~ UPPER(`nom`)")) assert.NoError(t, err) //dc.Label("nom") err = customers.AddColumn("email", datatable.String) assert.NoError(t, err) err = customers.AddColumn("ville", datatable.String) assert.NoError(t, err) customers.AppendRow(1, "Aimée", "Marechal", nil, "aime.marechal@example.com", "Paris") customers.AppendRow(2, "Esmée", "Lefort", nil, "esmee.lefort@example.com", "Lyon") customers.AppendRow(3, "Marine", "Prevost", nil, "m.prevost@example.com", "Lille") customers.AppendRow(4, "Luc", "Rolland", nil, "lucrolland@example.com", "Marseille") // Change structs assert.NoError(t, customers.RenameColumn("id", "Client ID")) customers.HideColumn("prenom") customers.HideColumn("nom") assert.Error(t, customers.RenameColumn("expr_nom", "nom")) assert.NoError(t, customers.RenameColumn("expr_nom", "Nom")) checkTable(t, customers, "Client ID", "Nom", "email", "ville", 1, "Aimée MARECHAL", "aime.marechal@example.com", "Paris", 2, "Esmée LEFORT", "esmee.lefort@example.com", "Lyon", 3, "Marine PREVOST", "m.prevost@example.com", "Lille", 4, "Luc ROLLAND", "lucrolland@example.com", "Marseille", ) return customers } func TestToTable(t *testing.T) { dt := sampleForExport(t) out := dt.ToTable() assert.NotNil(t, out) expected := `[ ["Client ID", "Nom", "email", "ville"], [1, "Aimée MARECHAL", "aime.marechal@example.com", "Paris"], [2, "Esmée LEFORT", "esmee.lefort@example.com", "Lyon"], [3, "Marine PREVOST", "m.prevost@example.com", "Lille"], [4, "Luc ROLLAND", "lucrolland@example.com", "Marseille"] ]` bytes, err := json.Marshal(out) assert.NoError(t, err) assert.JSONEq(t, expected, string(bytes)) out2 := dt.ToTable(datatable.ExportHidden(true)) assert.NotNil(t, out2) expected2 := `[ ["Client ID", "prenom", "nom", "Nom", "email", "ville"], [1, "Aimée", "Marechal", "Aimée MARECHAL", "aime.marechal@example.com", "Paris"], [2, "Esmée", "Lefort", "Esmée LEFORT", "esmee.lefort@example.com", "Lyon"], [3, "Marine", "Prevost", "Marine PREVOST", "m.prevost@example.com", "Lille"], [4, "Luc", "Rolland", "Luc ROLLAND", "lucrolland@example.com", "Marseille"] ]` bytes, err = json.Marshal(out2) assert.NoError(t, err) assert.JSONEq(t, expected2, string(bytes)) } func TestToMap(t *testing.T) { dt := sampleForExport(t) out := dt.ToMap() assert.NotNil(t, out) expected := `[ { "Client ID":1, "Nom":"Aimée MARECHAL", "email":"aime.marechal@example.com", "ville":"Paris" }, { "Client ID":2, "Nom":"Esmée LEFORT", "email":"esmee.lefort@example.com", "ville":"Lyon" }, { "Client ID":3, "Nom":"Marine PREVOST", "email":"m.prevost@example.com", "ville":"Lille" }, { "Client ID":4, "Nom":"Luc ROLLAND", "email":"lucrolland@example.com", "ville":"Marseille" } ]` bytes, err := json.Marshal(out) assert.NoError(t, err) assert.JSONEq(t, expected, string(bytes)) out2 := dt.ToMap(datatable.ExportHidden(true)) assert.NotNil(t, out2) expected2 := `[ { "Client ID":1, "prenom": "Aimée", "nom": "Marechal", "Nom":"Aimée MARECHAL", "email":"aime.marechal@example.com", "ville":"Paris" }, { "Client ID":2, "prenom": "Esmée", "nom": "Lefort", "Nom":"Esmée LEFORT", "email":"esmee.lefort@example.com", "ville":"Lyon" }, { "Client ID":3, "prenom": "Marine", "nom": "Prevost", "Nom":"Marine PREVOST", "email":"m.prevost@example.com", "ville":"Lille" }, { "Client ID":4, "prenom": "Luc", "nom": "Rolland", "Nom":"Luc ROLLAND", "email":"lucrolland@example.com", "ville":"Marseille" } ]` bytes, err = json.Marshal(out2) assert.NoError(t, err) assert.JSONEq(t, expected2, string(bytes)) } func TestToSchema(t *testing.T) { dt := sampleForExport(t) schema := dt.ToSchema() assert.NotNil(t, schema) assert.Equal(t, "Customers", schema.Name) assert.Equal(t, []datatable.SchemaColumn{ datatable.SchemaColumn{"Client ID", "NullInt"}, datatable.SchemaColumn{"Nom", "NullString"}, datatable.SchemaColumn{"email", "NullString"}, datatable.SchemaColumn{"ville", "NullString"}, }, schema.Columns) assert.Len(t, schema.Rows, 4) assert.Equal(t, []interface{}{1, "Aimée MARECHAL", "aime.marechal@example.com", "Paris"}, schema.Rows[0]) assert.Equal(t, []interface{}{2, "Esmée LEFORT", "esmee.lefort@example.com", "Lyon"}, schema.Rows[1]) assert.Equal(t, []interface{}{3, "Marine PREVOST", "m.prevost@example.com", "Lille"}, schema.Rows[2]) assert.Equal(t, []interface{}{4, "Luc ROLLAND", "lucrolland@example.com", "Marseille"}, schema.Rows[3]) schema2 := dt.ToSchema(datatable.ExportHidden(true)) assert.NotNil(t, schema2) assert.Equal(t, "Customers", schema2.Name) assert.Equal(t, []datatable.SchemaColumn{ datatable.SchemaColumn{"Client ID", "NullInt"}, datatable.SchemaColumn{"prenom", "NullString"}, datatable.SchemaColumn{"nom", "NullString"}, datatable.SchemaColumn{"Nom", "NullString"}, datatable.SchemaColumn{"email", "NullString"}, datatable.SchemaColumn{"ville", "NullString"}, }, schema2.Columns) assert.Len(t, schema2.Rows, 4) assert.Equal(t, []interface{}{1, "Aimée", "Marechal", "Aimée MARECHAL", "aime.marechal@example.com", "Paris"}, schema2.Rows[0]) assert.Equal(t, []interface{}{2, "Esmée", "Lefort", "Esmée LEFORT", "esmee.lefort@example.com", "Lyon"}, schema2.Rows[1]) assert.Equal(t, []interface{}{3, "Marine", "Prevost", "Marine PREVOST", "m.prevost@example.com", "Lille"}, schema2.Rows[2]) assert.Equal(t, []interface{}{4, "Luc", "Rolland", "Luc ROLLAND", "lucrolland@example.com", "Marseille"}, schema2.Rows[3]) } ================================================ FILE: go.mod ================================================ module github.com/datasweet/datatable go 1.12 require ( github.com/cespare/xxhash v1.1.0 github.com/datasweet/cast v1.2.0 github.com/datasweet/expr v1.3.0 github.com/olekukonko/tablewriter v0.0.4 github.com/pkg/errors v0.8.1 github.com/stretchr/testify v1.5.1 gonum.org/v1/gonum v0.7.0 ) ================================================ FILE: go.sum ================================================ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/datasweet/cast v1.2.0 h1:ApnOmjxZxR4r+LpBKP8lYrJULO/5hWPRCIvpH4TjLIY= github.com/datasweet/cast v1.2.0/go.mod h1:ByvG5xnUqdkDkxPp5AYlfvjuO87I/3R4xFy+Zi4SDdc= github.com/datasweet/expr v1.3.0 h1:4Anw5bjslDtXNOR2ZOScfCECCSoJqA491O9nSYpisLc= github.com/datasweet/expr v1.3.0/go.mod h1:CjkMBXNXUoZNrqglnyUKMf+j8FQxLYPFeWE/G40/7zk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk= github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= ================================================ FILE: hasher.go ================================================ package datatable import ( "bytes" "encoding/gob" "github.com/cespare/xxhash" ) var hasher = &hasherImpl{} type hasherImpl struct{} func (h *hasherImpl) Row(row Row, cols []string) uint64 { if row == nil { return 0 } buff := new(bytes.Buffer) enc := gob.NewEncoder(buff) for _, name := range cols { enc.Encode(row[name]) } return xxhash.Sum64(buff.Bytes()) } func (h *hasherImpl) Table(dt *DataTable, cols []string) map[uint64][]int { if dt == nil { return nil } mh := make(map[uint64][]int, 0) for i, row := range dt.Rows() { hash := h.Row(row, cols) mh[hash] = append(mh[hash], i) } return mh } ================================================ FILE: import/csv/import.go ================================================ package csv import ( "encoding/csv" "fmt" "io" "os" "github.com/datasweet/cast" "github.com/datasweet/datatable" "github.com/pkg/errors" ) // Import a csv func Import(name, path string, opt ...Option) (*datatable.DataTable, error) { options := Options{ IgnoreIfReadLineError: true, Comma: ',', TrimLeadingSpace: true, } for _, o := range opt { o(&options) } // Open the file file, err := os.Open(path) if err != nil { return nil, errors.Wrap(err, datatable.ErrOpenFile.Error()) } defer file.Close() reader := csv.NewReader(file) reader.Comma = options.Comma reader.Comment = options.Comment reader.LazyQuotes = options.LazyQuotes reader.TrimLeadingSpace = options.TrimLeadingSpace dt := datatable.New(name) line := 1 // Get columns names with headers if options.HasHeaders { rec, err := reader.Read() if err != nil { return nil, errors.Wrap(err, datatable.ErrCantReadHeaders.Error()) } if len(options.ColumnNames) == 0 { options.ColumnNames = append(options.ColumnNames, rec...) } line++ } for { rec, err := reader.Read() if err != nil { if err == io.EOF { if line == 1 { return nil, datatable.ErrNilDatas } break } if options.IgnoreIfReadLineError { continue } err := errors.Wrapf(err, "error line %d", line) return nil, errors.Wrap(err, datatable.ErrReadingLine.Error()) } // Do we have columns names ? if len(options.ColumnNames) == 0 { options.ColumnNames = make([]string, 0, len(rec)) for i := range rec { options.ColumnNames = append(options.ColumnNames, fmt.Sprintf("col %d", i+1)) } } // Do we have columns in datatable if dt.NumCols() == 0 { // Detect type if needed if len(options.ColumnTypes) == 0 { options.ColumnTypes = detectTypes(rec, options.DateFormats) } if len(options.ColumnNames) != len(options.ColumnTypes) { err := errors.Errorf("expected %d types, got %d", len(options.ColumnNames), len(options.ColumnTypes)) return nil, errors.Wrap(err, datatable.ErrWrongNumberOfTypes.Error()) } for i := range options.ColumnNames { if err := dt.AddColumn(options.ColumnNames[i], options.ColumnTypes[i], datatable.TimeFormats(options.DateFormats...)); err != nil { err = errors.Wrapf(err, "add column '%s' with type '%s'", options.ColumnNames[i], options.ColumnTypes[i]) return nil, errors.Wrap(err, datatable.ErrAddingColumn.Error()) } } } // conv => []interface{} cells := make([]interface{}, 0, len(rec)) for _, r := range rec { cells = append(cells, r) } dt.AppendRow(cells...) line++ } return dt, nil } func detectTypes(rec, dateformat []string) []datatable.ColumnType { ctypes := make([]datatable.ColumnType, 0, len(rec)) for _, r := range rec { if _, ok := cast.AsFloat64(r); ok { ctypes = append(ctypes, datatable.Float64) continue } if _, ok := cast.AsBool(r); ok { ctypes = append(ctypes, datatable.Bool) continue } if _, ok := cast.AsTime(r, dateformat...); ok { ctypes = append(ctypes, datatable.Time) continue } ctypes = append(ctypes, datatable.String) } return ctypes } ================================================ FILE: import/csv/import_test.go ================================================ package csv_test import ( "fmt" "os" "testing" "time" "github.com/datasweet/datatable" "github.com/datasweet/datatable/import/csv" "github.com/stretchr/testify/assert" ) func TestImport(t *testing.T) { dt, err := csv.Import("csv", "../../test/phone_data.csv", csv.HasHeader(true), csv.AcceptDate("02/01/06 15:04"), csv.AcceptDate("2006-01"), ) assert.NoError(t, err) assert.NotNil(t, dt) dt.Print(os.Stdout, datatable.PrintMaxRows(24)) dtc, err := dt.Aggregate(datatable.AggregateBy{Type: datatable.Count, Field: "index"}) assert.NoError(t, err) fmt.Println(dtc) groups, err := dt.GroupBy(datatable.GroupBy{ Name: "year", Type: datatable.Int, Keyer: func(row datatable.Row) (interface{}, bool) { if d, ok := row["date"]; ok { if tm, ok := d.(time.Time); ok { return tm.Year(), true } } return nil, false }, }) assert.NoError(t, err) out, err := groups.Aggregate( datatable.AggregateBy{Type: datatable.Sum, Field: "duration"}, datatable.AggregateBy{Type: datatable.CountDistinct, Field: "network"}, ) assert.NoError(t, err) fmt.Println(out) } ================================================ FILE: import/csv/options.go ================================================ package csv import "github.com/datasweet/datatable" // Options are options to import a csv type Options struct { HasHeaders bool ColumnNames []string // if len == 0 => take headers else "col #i" ColumnTypes []datatable.ColumnType // if len == 0 => detection IgnoreIfReadLineError bool Comma rune Comment rune LazyQuotes bool TrimLeadingSpace bool DateFormats []string } // Option is a setter type Option func(*Options) // HasHeader to retrieve column names on line #1 func HasHeader(v bool) Option { return func(opts *Options) { opts.HasHeaders = v } } // ColumnNames defines the column name func ColumnNames(v ...string) Option { return func(opts *Options) { opts.ColumnNames = v } } // ColumnTypes defines the column type func ColumnTypes(v ...datatable.ColumnType) Option { return func(opts *Options) { opts.ColumnTypes = v } } // IgnoreLineWithError to not stop the reading process if a line has an error func IgnoreLineWithError(v bool) Option { return func(opts *Options) { opts.IgnoreIfReadLineError = v } } // Comma is the field delimiter // Default to ',' func Comma(v rune) Option { return func(opts *Options) { opts.Comma = v } } // Comment if not 0, is the comment character. Lines beginning with the // Comment character without preceding whitespace are ignored. func Comment(v rune) Option { return func(opts *Options) { opts.Comment = v } } // LazyQuotes is true, a quote may appear in an unquoted field and a // non-doubled quote may appear in a quoted field. func LazyQuotes(v bool) Option { return func(opts *Options) { opts.LazyQuotes = v } } // TrimLeadingSpace is true, leading white space in a field is ignored. // This is done even if the field delimiter, Comma, is white space. func TrimLeadingSpace(v bool) Option { return func(opts *Options) { opts.TrimLeadingSpace = v } } // AcceptDate to accept a specific date format func AcceptDate(v string) Option { return func(opts *Options) { opts.DateFormats = append(opts.DateFormats, v) } } ================================================ FILE: join.go ================================================ package datatable import ( "regexp" "strings" "github.com/pkg/errors" ) // InnerJoin selects records that have matching values in both tables. // left datatable is used as reference datatable. // InnerJoin transforms an expr column to a raw column func (left *DataTable) InnerJoin(right *DataTable, on []JoinOn) (*DataTable, error) { return newJoinImpl(innerJoin, []*DataTable{left, right}, on).Compute() } // InnerJoin selects records that have matching values in both tables. // tables[0] is used as reference datatable. func InnerJoin(tables []*DataTable, on []JoinOn) (*DataTable, error) { return newJoinImpl(innerJoin, tables, on).Compute() } // LeftJoin returns all records from the left table (table1), and the matched records from the right table (table2). // The result is NULL from the right side, if there is no match. // LeftJoin transforms an expr column to a raw column func (left *DataTable) LeftJoin(right *DataTable, on []JoinOn) (*DataTable, error) { return newJoinImpl(leftJoin, []*DataTable{left, right}, on).Compute() } // LeftJoin the tables. // tables[0] is used as reference datatable. func LeftJoin(tables []*DataTable, on []JoinOn) (*DataTable, error) { return newJoinImpl(leftJoin, tables, on).Compute() } // RightJoin returns all records from the right table (table2), and the matched records from the left table (table1). // The result is NULL from the left side, when there is no match. // RightJoin transforms an expr column to a raw column func (left *DataTable) RightJoin(right *DataTable, on []JoinOn) (*DataTable, error) { return newJoinImpl(rightJoin, []*DataTable{left, right}, on).Compute() } // RightJoin the tables. // tables[0] is used as reference datatable. func RightJoin(tables []*DataTable, on []JoinOn) (*DataTable, error) { return newJoinImpl(rightJoin, tables, on).Compute() } // OuterJoin returns all records when there is a match in either left or right table // OuterJoin transforms an expr column to a raw column func (left *DataTable) OuterJoin(right *DataTable, on []JoinOn) (*DataTable, error) { return newJoinImpl(outerJoin, []*DataTable{left, right}, on).Compute() } // OuterJoin the tables. // tables[0] is used as reference datatable. func OuterJoin(tables []*DataTable, on []JoinOn) (*DataTable, error) { return newJoinImpl(outerJoin, tables, on).Compute() } type JoinOn struct { Table string Field string } var rgOn = regexp.MustCompile(`^(?:\[([^]]+)\]\.)?(?:\[([^]]+)\])$`) // On creates a "join on" expression // ie, as SQL, SELECT * FROM A INNER JOIN B ON B.id = A.user_id // Syntax: "[table].[field]", "field" func On(fields ...string) []JoinOn { var jon []JoinOn for _, f := range fields { matches := rgOn.FindStringSubmatch(f) switch len(matches) { case 0: jon = append(jon, JoinOn{Table: "*", Field: f}) case 3: t := matches[1] if len(t) == 0 { t = "*" } jon = append(jon, JoinOn{Table: t, Field: matches[2]}) default: return nil } } return jon } // Using creates a "join using" expression // ie, as SQL, SELECT * FROM A INNER JOIN B USING 'field' func Using(fields ...string) []JoinOn { var jon []JoinOn for _, f := range fields { jon = append(jon, JoinOn{Table: "*", Field: f}) } return jon } type joinType uint8 const ( innerJoin joinType = iota leftJoin rightJoin outerJoin ) func colname(dt *DataTable, col string) string { var sb strings.Builder sb.WriteString(dt.Name()) sb.WriteString(".") sb.WriteString(col) return sb.String() } type joinClause struct { table *DataTable mcols map[string][]string on []string includeOnCols bool cmapper [][2]string // [initial, output] hashtable map[uint64][]int consumed map[int]bool } func (jc *joinClause) copyColumnsTo(out *DataTable) error { if out == nil { return ErrNilOutputDatatable } mon := make(map[string]bool, len(jc.on)) for _, o := range jc.on { mon[o] = true } for _, col := range jc.table.cols { name := col.name cname := name if _, found := mon[name]; found { if !jc.includeOnCols { continue } } else if v, ok := jc.mcols[name]; ok && len(v) > 1 { // commons col between table for _, tn := range v { if tn == jc.table.name { cname = colname(jc.table, name) break } } } ccpy := col.emptyCopy() ccpy.name = cname if err := out.addColumn(ccpy); err != nil { return err } jc.cmapper = append(jc.cmapper, [2]string{name, cname}) } return nil } func (jc *joinClause) initHashTable() { jc.hashtable = hasher.Table(jc.table, jc.on) jc.consumed = make(map[int]bool, jc.table.NumRows()) } type joinImpl struct { mode joinType tables []*DataTable on []JoinOn clauses []*joinClause mcols map[string][]string } func newJoinImpl(mode joinType, tables []*DataTable, on []JoinOn) *joinImpl { return &joinImpl{ mode: mode, tables: tables, on: on, } } func (j *joinImpl) Compute() (*DataTable, error) { if err := j.checkInput(); err != nil { return nil, err } j.initColMapper() out := j.tables[0] for i := 1; i < len(j.tables); i++ { jdt, err := j.join(out, j.tables[i]) if err != nil { return nil, err } out = jdt } if out == nil { return nil, ErrNoOutput } return out, nil } func (j *joinImpl) checkInput() error { if len(j.tables) < 2 { return ErrNotEnoughDatatables } for i, t := range j.tables { if t == nil || len(t.Name()) == 0 || t.NumCols() == 0 { err := errors.Errorf("table #%d is nil", i) return errors.Wrap(err, ErrNilTable.Error()) } } if len(j.on) == 0 { return ErrNoOnClauses } for i, o := range j.on { if len(o.Field) == 0 { err := errors.Errorf("on #%d is nil", i) return errors.Wrap(err, ErrOnClauseIsNil.Error()) } } return nil } func (j *joinImpl) initColMapper() { mcols := make(map[string][]string) for _, t := range j.tables { for _, name := range t.cols { mcols[name.name] = append(mcols[name.name], t.Name()) } } j.mcols = mcols } func (j *joinImpl) join(left, right *DataTable) (*DataTable, error) { if left == nil { err := errors.New("left is nil datatable") return nil, errors.Wrap(err, ErrNilDatatable.Error()) } if right == nil { err := errors.New("right is nil datatable") return nil, errors.Wrap(err, ErrNilDatatable.Error()) } clauses := [2]*joinClause{ &joinClause{ table: left, mcols: j.mcols, includeOnCols: true, }, &joinClause{ table: right, mcols: j.mcols, }, } // find on clauses for _, o := range j.on { if o.Table == left.Name() { clauses[0].on = append(clauses[0].on, o.Field) continue } if o.Table == right.Name() { clauses[1].on = append(clauses[1].on, o.Field) continue } if o.Table == "*" || len(o.Table) == 0 { clauses[0].on = append(clauses[0].on, o.Field) clauses[1].on = append(clauses[1].on, o.Field) } } // create output out := New(left.Name()) for _, clause := range clauses { if err := clause.copyColumnsTo(out); err != nil { return nil, err } } // mode var ref, join *joinClause switch j.mode { case innerJoin, leftJoin, outerJoin: ref, join = clauses[0], clauses[1] case rightJoin: ref, join = clauses[1], clauses[0] default: err := errors.Errorf("unknown mode '%v'", j.mode) return nil, errors.Wrap(err, ErrUnknownMode.Error()) } join.initHashTable() // Copy rows for _, refrow := range ref.table.Rows(ExportHidden(true)) { // Create hash hash := hasher.Row(refrow, ref.on) // Have we same hash in jointable ? if indexes, ok := join.hashtable[hash]; ok { for _, idx := range indexes { joinrow := join.table.Row(idx, ExportHidden(true)) row := out.NewRow() for _, cm := range ref.cmapper { row[cm[1]] = refrow.Get(cm[0]) } for _, cm := range join.cmapper { row[cm[1]] = joinrow.Get(cm[0]) } join.consumed[idx] = true out.Append(row) } } else if j.mode != innerJoin { row := make(Row, len(refrow)) for _, cm := range ref.cmapper { row[cm[1]] = refrow.Get(cm[0]) } out.Append(row) } } // out.Print(os.Stdout, PrintColumnType(false)) // Outer: we must copy rows not consummed in right (join) table if j.mode == outerJoin { for i, joinrow := range join.table.Rows() { if b, ok := join.consumed[i]; ok && b { continue } row := make(Row, len(joinrow)) for _, cm := range join.cmapper { row[cm[1]] = joinrow.Get(cm[0]) } out.Append(row) } } return out, nil } ================================================ FILE: join_test.go ================================================ package datatable_test import ( "fmt" "testing" "time" "github.com/datasweet/datatable" "github.com/stretchr/testify/assert" ) // https://sql.sh/cours/jointures/inner-join func sampleForJoin() (*datatable.DataTable, *datatable.DataTable) { customers := datatable.New("Customers") customers.AddColumn("id", datatable.Int) customers.AddColumn("prenom", datatable.String) customers.AddColumn("nom", datatable.String) customers.AddColumn("email", datatable.String) customers.AddColumn("ville", datatable.String) // customers.AddColumn("concat", datatable.String, datatable.Expr("CONCAT(`prenom`,`nom`)")) customers.AppendRow(1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris") customers.AppendRow(2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon") customers.AppendRow(3, "Marine", "Prevost", "m.prevost@example.com", "Lille") customers.AppendRow(4, "Luc", "Rolland", "lucrolland@example.com", "Marseille") orders := datatable.New("Orders") orders.AddColumn("user_id", datatable.Int, datatable.Values(1, 1, 2, 3, 5)) orders.AddColumn("date_achat", datatable.Time, datatable.Values("2013-01-23", "2013-02-14", "2013-02-17", "2013-02-21", "2013-03-02")) orders.AddColumn("num_facture", datatable.String, datatable.Values("A00103", "A00104", "A00105", "A00106", "A00107")) orders.AddColumn("prix_total", datatable.Float64, datatable.Values(203.14, 124.00, 149.45, 235.35, 47.58)) return customers, orders } func TestJoinOn(t *testing.T) { on := datatable.On("[customers].[id]") assert.NotNil(t, on) assert.Len(t, on, 1) assert.Equal(t, "customers", on[0].Table) assert.Equal(t, "id", on[0].Field) on = datatable.On("[id]") assert.NotNil(t, on) assert.Len(t, on, 1) assert.Equal(t, "*", on[0].Table) assert.Equal(t, "id", on[0].Field) on = datatable.On("id") assert.NotNil(t, on) assert.Len(t, on, 1) assert.Equal(t, "*", on[0].Table) assert.Equal(t, "id", on[0].Field) on = datatable.On("customers.[id]") assert.NotNil(t, on) assert.Len(t, on, 1) assert.Equal(t, "*", on[0].Table) assert.Equal(t, "customers.[id]", on[0].Field) } func TestInnerJoin(t *testing.T) { customers, orders := sampleForJoin() customers.AddColumn("concat", datatable.String, datatable.Expr("concat(`prenom`, `nom`)")) dt, err := customers.InnerJoin(orders, datatable.On("[Customers].[id]", "[Orders].[user_id]")) assert.NoError(t, err) assert.NotNil(t, dt) checkTable(t, dt, "id", "prenom", "nom", "email", "ville", "concat", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", "AiméeMarechal", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", "AiméeMarechal", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", "EsméeLefort", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", "MarinePrevost", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, ) } func TestLeftJoin(t *testing.T) { customers, orders := sampleForJoin() dt, err := customers.LeftJoin(orders, datatable.On("[Customers].[id]", "[Orders].[user_id]")) assert.NoError(t, err) assert.NotNil(t, dt) checkTable(t, dt, "id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, 4, "Luc", "Rolland", "lucrolland@example.com", "Marseille", nil, nil, nil, ) } func TestRightJoin(t *testing.T) { customers, orders := sampleForJoin() dt, err := customers.RightJoin(orders, datatable.On("[Customers].[id]", "[Orders].[user_id]")) assert.NoError(t, err) assert.NotNil(t, dt) checkTable(t, dt, "id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, nil, nil, nil, nil, nil, time.Date(2013, time.March, 2, 0, 0, 0, 0, time.UTC), "A00107", 47.58, ) } func TestOuterJoin(t *testing.T) { customers, orders := sampleForJoin() dt, err := customers.OuterJoin(orders, datatable.On("[Customers].[id]", "[Orders].[user_id]")) assert.NoError(t, err) assert.NotNil(t, dt) checkTable(t, dt, "id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, 4, "Luc", "Rolland", "lucrolland@example.com", "Marseille", nil, nil, nil, nil, nil, nil, nil, nil, time.Date(2013, time.March, 2, 0, 0, 0, 0, time.UTC), "A00107", 47.58, ) } func TestInnerJoinWithExprOnHidden(t *testing.T) { customers, orders := sampleForJoin() customers.AddColumn("id2", datatable.Int, datatable.Expr("`id`+100")) orders.AddColumn("user_id2", datatable.Int, datatable.Expr("`user_id`+100")) customers.HideColumn("id") dt, err := customers.InnerJoin(orders, datatable.On("[Customers].[id2]", "[Orders].[user_id2]")) assert.NoError(t, err) assert.NotNil(t, dt) fmt.Println(dt) checkTable(t, dt, "prenom", "nom", "email", "ville", "id2", "user_id", "date_achat", "num_facture", "prix_total", "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", 102, 2, time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, "Marine", "Prevost", "m.prevost@example.com", "Lille", 103, 3, time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, ) } func TestLeftJoinWithExpr(t *testing.T) { customers, orders := sampleForJoin() customers.AddColumn("id2", datatable.Int, datatable.Expr("`id`+100")) orders.AddColumn("user_id2", datatable.Int, datatable.Expr("`user_id`+100")) dt, err := customers.LeftJoin(orders, datatable.On("[Customers].[id2]", "[Orders].[user_id2]")) assert.NoError(t, err) assert.NotNil(t, dt) fmt.Println(dt) checkTable(t, dt, "id", "prenom", "nom", "email", "ville", "id2", "user_id", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", 102, 2, time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", 103, 3, time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, 4, "Luc", "Rolland", "lucrolland@example.com", "Marseille", 104, nil, nil, nil, nil, ) } func TestRightJoinWithExpr(t *testing.T) { customers, orders := sampleForJoin() customers.AddColumn("id2", datatable.Int, datatable.Expr("`id`+100")) orders.AddColumn("user_id2", datatable.Int, datatable.Expr("`user_id`+100")) dt, err := customers.RightJoin(orders, datatable.On("[Customers].[id2]", "[Orders].[user_id2]")) assert.NoError(t, err) assert.NotNil(t, dt) fmt.Println(dt) checkTable(t, dt, "id", "prenom", "nom", "email", "ville", "id2", "user_id", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", 101, 1, time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", 102, 2, time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", 103, 3, time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, nil, nil, nil, nil, nil, nil, 5, time.Date(2013, time.March, 2, 0, 0, 0, 0, time.UTC), "A00107", 47.58, ) } func TestJoinWithColumnName(t *testing.T) { customers, orders := sampleForJoin() assert.NoError(t, customers.RenameColumn("id", "ClientID")) dt, err := customers.InnerJoin(orders, datatable.On("[Customers].[ClientID]", "[Orders].[user_id]")) assert.NoError(t, err) assert.NotNil(t, dt) checkTable(t, dt, "ClientID", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, ) } ================================================ FILE: mutate_column.go ================================================ package datatable import ( "strings" "github.com/datasweet/expr" "github.com/pkg/errors" ) func (t *DataTable) addColumn(col *column) error { if col == nil { return ErrNilColumn } // Check name if len(col.name) == 0 { return ErrNilColumnName } if c := t.Column(col.name); c != nil { err := errors.Errorf("column '%s' already exists", col.name) return errors.Wrap(err, ErrColumnAlreadyExists.Error()) } // Check typ if len(col.typ) == 0 { return ErrNilColumnType } // Check formula if len(col.formulae) > 0 { parsed, err := expr.Parse(col.formulae) if err != nil { return errors.Wrapf(err, ErrFormulaeSyntax.Error()) } col.expr = parsed t.hasExpr = true } // Check serie if col.serie == nil { return ErrNilSerie } ln := col.serie.Len() if ln < t.nrows { col.serie.Grow(t.nrows - ln) } else if ln > t.nrows { size := ln - t.nrows for _, col := range t.cols { col.serie.Grow(size) } t.nrows = ln } t.cols = append(t.cols, col) t.dirty = true return nil } // AddColumn to datatable with a serie of T func (t *DataTable) AddColumn(name string, ctyp ColumnType, opt ...ColumnOption) error { var options ColumnOptions for _, o := range opt { o(&options) } // create serie based on ctyp sr, err := newColumnSerie(ctyp, options) if err != nil { return errors.Wrap(err, ErrCreateSerie.Error()) } return t.addColumn(&column{ name: strings.TrimSpace(name), typ: ctyp, serie: sr, hidden: options.Hidden, formulae: strings.TrimSpace(options.Expr), }) } // RenameColumn to rename a column func (t *DataTable) RenameColumn(old, name string) error { name = strings.TrimSpace(name) if len(name) == 0 { err := errors.New("you must provided a column name") return errors.Wrap(err, ErrNilColumnName.Error()) } if c := t.Column(name); c != nil { err := errors.Errorf("column '%s' already exists", name) return errors.Wrap(err, ErrColumnAlreadyExists.Error()) } if col := t.Column(old); col != nil { col.(*column).name = name return nil } err := errors.Errorf("column '%s' does not exist", name) return errors.Wrap(err, ErrColumnNotFound.Error()) } // HideAll to hides all column // a hidden column will not be exported func (t *DataTable) HideAll() { for _, col := range t.cols { col.hidden = true } } // HideColumn hides a column // a hidden column will not be exported func (t *DataTable) HideColumn(name string) { if c := t.Column(name); c != nil { (c.(*column)).hidden = true } } // ShowAll to show all column // a shown column will be exported func (t *DataTable) ShowAll() { for _, col := range t.cols { col.hidden = false } } // ShowColumn shows a column // a shown column will be exported func (t *DataTable) ShowColumn(name string) { if c := t.Column(name); c != nil { (c.(*column)).hidden = false } } // SwapColumn to swap 2 columns func (t *DataTable) SwapColumn(a, b string) error { i := t.ColumnIndex(a) if i < 0 { err := errors.Errorf("column '%s' not found", a) return errors.Wrap(err, ErrColumnNotFound.Error()) } j := t.ColumnIndex(b) if j < 0 { err := errors.Errorf("column '%s' not found", b) return errors.Wrap(err, ErrColumnNotFound.Error()) } t.cols[i], t.cols[j] = t.cols[j], t.cols[i] return nil } ================================================ FILE: mutate_column_test.go ================================================ package datatable_test import ( "testing" "github.com/datasweet/datatable" ) func TestSwapColumn(t *testing.T) { tb := datatable.New("test") tb.AddColumn("champ", datatable.String, datatable.Values("Malzahar", "Xerath", "Teemo")) tb.AddColumn("champion", datatable.String, datatable.Expr("upper(`champ`)")) tb.AddColumn("win", datatable.Int, datatable.Values(10, 20, 666)) tb.AddColumn("loose", datatable.Int, datatable.Values(6, 5, 666)) tb.AddColumn("winRate", datatable.Float64, datatable.Expr("(`win` * 100 / (`win` + `loose`))")) checkTable(t, tb, "champ", "champion", "win", "loose", "winRate", "Malzahar", "MALZAHAR", 10, 6, 62.5, "Xerath", "XERATH", 20, 5, 80.0, "Teemo", "TEEMO", 666, 666, 50.0, ) tb.SwapColumn("champion", "winRate") checkTable(t, tb, "champ", "winRate", "win", "loose", "champion", "Malzahar", 62.5, 10, 6, "MALZAHAR", "Xerath", 80.0, 20, 5, "XERATH", "Teemo", 50.0, 666, 666, "TEEMO", ) } ================================================ FILE: mutate_row.go ================================================ package datatable import ( "github.com/pkg/errors" ) // NewRow create a new row func (t *DataTable) NewRow() Row { r := make(Row) return r } // Append rows to the table func (t *DataTable) Append(row ...Row) { for _, r := range row { if r == nil { continue } for _, col := range t.cols { if !col.IsComputed() { if cell, ok := r[col.Name()]; ok { col.serie.Append(cell) continue } } col.serie.Grow(1) } t.nrows++ } t.dirty = true } // AppendRow creates a new row and append cells to this row func (t *DataTable) AppendRow(v ...interface{}) error { if len(v) != len(t.cols) { err := errors.Errorf("length mismatch: expected %d elements, values have %d elements", len(t.cols), len(v)) return errors.Wrap(err, ErrLengthMismatch.Error()) } for i, col := range t.cols { if col.IsComputed() { col.serie.Grow(1) } else { col.serie.Append(v[i]) } } t.nrows++ t.dirty = true return nil } // SwapRow in table func (t *DataTable) SwapRow(i, j int) { for _, col := range t.cols { col.serie.Swap(i, j) } } // Grow the table by size func (t *DataTable) Grow(size int) { for _, col := range t.cols { col.serie.Grow(size) } } // Update the row at index func (t *DataTable) Update(at int, row Row) error { if row == nil { row = make(Row, 0) } for _, col := range t.cols { if col.IsComputed() { continue } cell, ok := row[col.name] if ok { if err := col.serie.Set(at, cell); err != nil { err := errors.Wrapf(err, "col %s", col.name) return errors.Wrap(err, ErrUpdateRow.Error()) } continue } if err := col.serie.Set(at, nil); err != nil { err := errors.Wrapf(err, "col %s", col.name) return errors.Wrap(err, ErrUpdateRow.Error()) } } return nil } ================================================ FILE: mutate_row_test.go ================================================ package datatable_test import ( "testing" "github.com/datasweet/datatable" ) func TestSwapRow(t *testing.T) { tb := datatable.New("test") tb.AddColumn("champ", datatable.String, datatable.Values("Malzahar", "Xerath", "Teemo")) tb.AddColumn("champion", datatable.String, datatable.Expr("upper(`champ`)")) tb.AddColumn("win", datatable.Int, datatable.Values(10, 20, 666)) tb.AddColumn("loose", datatable.Int, datatable.Values(6, 5, 666)) tb.AddColumn("winRate", datatable.Float64, datatable.Expr("(`win` * 100 / (`win` + `loose`))")) checkTable(t, tb, "champ", "champion", "win", "loose", "winRate", "Malzahar", "MALZAHAR", 10, 6, 62.5, "Xerath", "XERATH", 20, 5, 80.0, "Teemo", "TEEMO", 666, 666, 50.0, ) tb.SwapRow(0, 2) checkTable(t, tb, "champ", "champion", "win", "loose", "winRate", "Teemo", "TEEMO", 666, 666, 50.0, "Xerath", "XERATH", 20, 5, 80.0, "Malzahar", "MALZAHAR", 10, 6, 62.5, ) } ================================================ FILE: row.go ================================================ package datatable import ( "bytes" "encoding/gob" "github.com/cespare/xxhash" ) // Row contains a row relative to columns type Row map[string]interface{} // Set cell func (r Row) Set(k string, v interface{}) Row { r[k] = v return r } // Get cell func (r Row) Get(k string) interface{} { // Check colName exists if v, ok := r[k]; ok { return v } return nil } // Hash computes the hash code from this datarow // can be used to filter the table (distinct rows) func (r Row) Hash() uint64 { buff := new(bytes.Buffer) enc := gob.NewEncoder(buff) for _, v := range r { enc.Encode(v) } return xxhash.Sum64(buff.Bytes()) } ================================================ FILE: select.go ================================================ package datatable // Subset selects rows at index with size func (t *DataTable) Subset(at, size int) *DataTable { cpy := t.EmptyCopy() for i, col := range t.cols { cpy.cols[i].serie = col.serie.Subset(at, size) } if len(cpy.cols) > 0 { cpy.nrows = cpy.cols[0].serie.Len() } return cpy } // Head selects {size} first rows func (t *DataTable) Head(size int) *DataTable { return t.Subset(0, size) } // Tail selects {size} last rows func (t *DataTable) Tail(size int) *DataTable { return t.Subset(t.nrows-size, size) } ================================================ FILE: serie/converters.go ================================================ package serie import ( "reflect" "github.com/datasweet/cast" ) // AsFloat64 to converts a serie to a serie of float64 // Used for some statistics func AsFloat64(s Serie, missing *float64) Serie { if s == nil || s.Len() == 0 { return Float64() // empty list of floats } switch kind := s.Type().Kind(); kind { case reflect.Float64: return s default: ln := s.Len() arr := make([]float64, 0, ln) for i := 0; i < ln; i++ { if f, ok := cast.AsFloat64(s.Get(i)); ok { arr = append(arr, f) continue } if missing != nil { arr = append(arr, *missing) } } sf := Float64() (sf.(*serie)).slice = reflect.ValueOf(arr) return sf } } ================================================ FILE: serie/copy.go ================================================ package serie import ( "reflect" ) func (s *serie) makeEmptyCopy(capacity int) *serie { return &serie{ typ: s.typ, converter: s.converter, comparer: s.comparer, interfacer: s.interfacer, slice: reflect.MakeSlice(reflect.SliceOf(s.typ), 0, capacity), } } func (s *serie) EmptyCopy() Serie { return s.makeEmptyCopy(0) } func (s *serie) Copy() Serie { cnt := s.Len() cpy := &serie{ typ: s.typ, converter: s.converter, comparer: s.comparer, interfacer: s.interfacer, slice: reflect.MakeSlice(reflect.SliceOf(s.typ), cnt, cnt), } reflect.Copy(cpy.slice, s.slice) return cpy } ================================================ FILE: serie/copy_test.go ================================================ package serie_test import ( "testing" "github.com/datasweet/datatable/serie" "github.com/stretchr/testify/assert" ) func TestCopy(t *testing.T) { original := serie.Int(1, 2, 3, 4, 5, 6, 7, 8, 9) cpy := original.Copy() assert.NotSame(t, original, cpy) assert.Equal(t, original.Type(), cpy.Type()) assert.Equal(t, original.Len(), cpy.Len()) assertSerieEq(t, cpy, 1, 2, 3, 4, 5, 6, 7, 8, 9) original.Set(4, 50) assertSerieEq(t, original, 1, 2, 3, 4, 50, 6, 7, 8, 9) assertSerieEq(t, cpy, 1, 2, 3, 4, 5, 6, 7, 8, 9) } func TestEmptyCopy(t *testing.T) { original := serie.Int(1, 2, 3, 4, 5, 6, 7, 8, 9) cpy := original.EmptyCopy() assert.NotSame(t, original, cpy) assert.Equal(t, original.Type(), cpy.Type()) assert.Equal(t, 9, original.Len()) assert.Equal(t, 0, cpy.Len()) } ================================================ FILE: serie/errors.go ================================================ package serie import ( "github.com/pkg/errors" ) // Errors in mutate.go var ( ErrOutOfRange = errors.New("out of range") ErrCantFlattenSliceWithSet = errors.New("can't flatten slice with set") ErrGrowSizeMustBeStriclyPositive = errors.New("grow: size must be > 0") ErrShrinkSizeMustBeStriclyPositive = errors.New("shrink: size must be > 0") ErrShrinkSizeMustBeLesserThanLen = errors.New("shrink: size must be < len") ErrConcatTypeMismatch = errors.New("concat: type mismatch") ) ================================================ FILE: serie/iterate.go ================================================ package serie // Iterator to creates a new iterator from the serie func (s *serie) Iterator() Iterator { return &serieIterator{ current: -1, serie: s, } } // Iterator defines an iterator // https://docs.microsoft.com/en-us/dotnet/api/system.collections.ienumerator.reset?view=netcore-3.1 type Iterator interface { Next() bool Current() interface{} Reset() } type serieIterator struct { current int serie *serie } func (it *serieIterator) Next() bool { it.current++ if it.current >= it.serie.Len() { return false } return true } func (it *serieIterator) Current() interface{} { return it.serie.Get(it.current) } func (it *serieIterator) Reset() { it.current = -1 } ================================================ FILE: serie/iterate_test.go ================================================ package serie_test import ( "testing" "github.com/datasweet/datatable/serie" "github.com/stretchr/testify/assert" ) func TestIterate(t *testing.T) { xs := []float64{ 32.32, 56.98, 21.52, 44.32, 55.63, 13.75, 43.47, 43.34, 12.34, } s := serie.Float64(xs) index := 0 for it := s.Iterator(); it.Next(); { assert.Equal(t, xs[index], it.Current()) index++ } } ================================================ FILE: serie/mutate.go ================================================ package serie import ( "reflect" "github.com/pkg/errors" ) func (s *serie) asValue(i interface{}) []reflect.Value { in := i if cs, ok := in.(Serie); ok { in = cs.Slice() } rv := reflect.ValueOf(in) kind := rv.Kind() switch kind { case reflect.Slice, reflect.Array: arr := make([]reflect.Value, 0, rv.Len()) for j := 0; j < rv.Len(); j++ { arr = append(arr, s.converter.Call([]reflect.Value{rv.Index(j)})...) } return arr case reflect.Invalid: // case "nil" return s.converter.Call([]reflect.Value{reflect.Zero(s.typ)}) default: return s.converter.Call([]reflect.Value{rv}) } } // Append values to the serie. func (s *serie) Append(v ...interface{}) { values := make([]reflect.Value, 0, len(v)) for _, val := range v { values = append(values, s.asValue(val)...) } s.slice = reflect.Append(s.slice, values...) } // Prepend values to the serie func (s *serie) Prepend(v ...interface{}) error { return s.Insert(0, v...) } // Insert values to the serie at index func (s *serie) Insert(at int, v ...interface{}) (err error) { n := s.Len() if at < 0 || ((at > 0 || n > 0) && at >= n) { err := errors.Errorf("insert at [%d]: index out of range with length %d", at, n) return errors.Wrap(err, ErrOutOfRange.Error()) } values := make([]reflect.Value, 0, len(v)) for _, val := range v { values = append(values, s.asValue(val)...) } if len(values) == 0 { return nil } for i := 0; i < len(values); i++ { s.slice = reflect.Append(s.slice, reflect.Zero(s.typ)) } // Refresh len n = s.Len() reflect.Copy(s.slice.Slice(at+len(values), n), s.slice.Slice(at, n)) for i, rv := range values { s.slice.Index(i + at).Set(rv) } return nil } // Set a value at index func (s *serie) Set(at int, v interface{}) error { if at < 0 || at >= s.Len() { err := errors.Errorf("set at [%d]: index out of range with length %d", at, s.Len()) return errors.Wrap(err, ErrOutOfRange.Error()) } values := s.asValue(v) if len(values) != 1 { err := errors.Errorf("set at [%d]: can't flatten slice with set", at) return errors.Wrap(err, ErrCantFlattenSliceWithSet.Error()) } s.slice.Index(at).Set(values[0]) return nil } // Delete a value at index func (s *serie) Delete(at int) error { cnt := s.Len() if at < 0 || at >= cnt { err := errors.Errorf("delete at [%d]: index out of range with length %d", at, cnt) return errors.Wrap(err, ErrCantFlattenSliceWithSet.Error()) } if at < cnt-1 { reflect.Copy(s.slice.Slice(at, cnt), s.slice.Slice(at+1, cnt)) } s.slice = s.slice.Slice(0, cnt-1) return nil } // Grow the serie with size // Grow will create zero value func (s *serie) Grow(size int) error { if size < 0 { err := errors.Errorf("grow: size '%d' must be > 0", size) return errors.Wrap(err, ErrGrowSizeMustBeStriclyPositive.Error()) } for i := 0; i < size; i++ { s.slice = reflect.Append(s.slice, reflect.Zero(s.typ)) } return nil } // Shrink the serie with size func (s *serie) Shrink(size int) error { if size < 0 { err := errors.Errorf("shrink: size '%d' must be > 0", size) return errors.Wrap(err, ErrShrinkSizeMustBeStriclyPositive.Error()) } cnt := s.Len() if size > cnt { err := errors.Errorf("shrink: size '%d' must be < length '%d'", size, cnt) return errors.Wrap(err, ErrShrinkSizeMustBeLesserThanLen.Error()) } s.slice = s.slice.Slice(0, cnt-size) return nil } // Concat the serie (mutate) with others series // series provided must be the same type as the source serie func (s *serie) Concat(serie ...Serie) error { if len(serie) == 0 { return nil } for i, other := range serie { if other.Type() != s.Type() { err := errors.Errorf("concat: serie #%d is not the same type as source", i) return errors.Wrap(err, ErrConcatTypeMismatch.Error()) } s.Append(other.Slice()) } return nil } func (s *serie) Clear() { s.slice = reflect.MakeSlice(reflect.SliceOf(s.typ), 0, 0) } ================================================ FILE: serie/mutate_test.go ================================================ package serie_test import ( "testing" "github.com/datasweet/datatable/serie" "github.com/stretchr/testify/assert" ) func TestAppend(t *testing.T) { s := serie.Int() assertSerieEq(t, s) s.Append(1, 2, 3, 4, "5") assertSerieEq(t, s, 1, 2, 3, 4, 5) s.Append(nil, 6, 7, "8", 9, 10) assertSerieEq(t, s, 1, 2, 3, 4, 5, 0, 6, 7, 8, 9, 10) } func TestPrepend(t *testing.T) { s := serie.Int() assertSerieEq(t, s) assert.NoError(t, s.Prepend(1, 2, 3, 4, 5)) assertSerieEq(t, s, 1, 2, 3, 4, 5) assert.NoError(t, s.Prepend(-4, -3, -2, -1, 0)) assertSerieEq(t, s, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5) } func TestInsert(t *testing.T) { s := serie.Int(1, 2, 3, 4, 5) assertSerieEq(t, s, 1, 2, 3, 4, 5) assert.NoError(t, s.Insert(2, 7, 8, 9, 10)) assertSerieEq(t, s, 1, 2, 7, 8, 9, 10, 3, 4, 5) assert.Error(t, s.Insert(-1, 7, 8, 9, 10)) assert.Error(t, s.Insert(101, 7, 8, 9, 10)) } func TestSet(t *testing.T) { s := serie.Int(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assert.Error(t, s.Set(-1, 100)) assert.Error(t, s.Set(10, 100)) assert.Error(t, s.Set(0, []int{0, 1, 2, 4})) assert.NoError(t, s.Set(5, 555)) assertSerieEq(t, s, 0, 1, 2, 3, 4, 555, 6, 7, 8, 9) assert.NoError(t, s.Set(9, 999)) assertSerieEq(t, s, 0, 1, 2, 3, 4, 555, 6, 7, 8, 999) assert.NoError(t, s.Set(0, -5)) assertSerieEq(t, s, -5, 1, 2, 3, 4, 555, 6, 7, 8, 999) } func TestDelete(t *testing.T) { s := serie.Int(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assert.Error(t, s.Delete(-1)) assert.Error(t, s.Delete(10)) assert.NoError(t, s.Delete(5)) assertSerieEq(t, s, 0, 1, 2, 3, 4, 6, 7, 8, 9) assert.NoError(t, s.Delete(8)) assertSerieEq(t, s, 0, 1, 2, 3, 4, 6, 7, 8) assert.NoError(t, s.Delete(0)) assertSerieEq(t, s, 1, 2, 3, 4, 6, 7, 8) } func TestGrow(t *testing.T) { s := serie.Int(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assert.Error(t, s.Grow(-5)) assert.NoError(t, s.Grow(5)) assertSerieEq(t, s, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0) s = serie.IntN(0, 1, 2, 3, 4, 5) assertSerieEq(t, s, 0, 1, 2, 3, 4, 5) assert.NoError(t, s.Grow(5)) assertSerieEq(t, s, 0, 1, 2, 3, 4, 5, nil, nil, nil, nil, nil) } func TestShrink(t *testing.T) { s := serie.Int(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assert.Error(t, s.Shrink(-5)) assert.Error(t, s.Shrink(11)) assert.NoError(t, s.Shrink(5)) assertSerieEq(t, s, 0, 1, 2, 3, 4) assert.NoError(t, s.Shrink(5)) assertSerieEq(t, s) } func TestConcat(t *testing.T) { s := serie.Int(0, 1, 2, 3, 4) assert.Error(t, s.Concat(serie.IntN(-1, -2, nil))) assert.NoError(t, s.Concat(serie.Int(6, 7, 8, 9, 10))) assertSerieEq(t, s, 0, 1, 2, 3, 4, 6, 7, 8, 9, 10) s = serie.StringN("Léon", "Marie", "Sophie", "Marcel") assertSerieEq(t, s, "Léon", "Marie", "Sophie", "Marcel") assert.NoError(t, s.Concat(serie.StringN("Marion", "Paul", "Marie", "Marcel"))) assertSerieEq(t, s, "Léon", "Marie", "Sophie", "Marcel", "Marion", "Paul", "Marie", "Marcel") } func TestClear(t *testing.T) { s := serie.Int(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assert.Equal(t, 10, s.Len()) s.Clear() assert.Equal(t, 0, s.Len()) } ================================================ FILE: serie/select.go ================================================ package serie import ( "reflect" ) // Head returns the first {size} rows of the serie func (s *serie) Head(size int) Serie { return s.Subset(0, size) } // Head returns the last {size} rows of the serie func (s *serie) Tail(size int) Serie { return s.Subset(s.Len()-size, size) } // Subset returns the a subset {at} index and with {size} func (s *serie) Subset(at, size int) Serie { cpy := s.makeEmptyCopy(0) ln := s.Len() if at < 0 || at >= ln || size <= 0 { return cpy } to := at + size if to > ln { to = ln } cpy.slice = s.slice.Slice(at, to) return cpy } // Filter the series with a predicate // Predicate must be func(T) bool func (s *serie) Filter(predicate interface{}) Serie { // Check predicate // must be func(T) bool if predicate == nil { panic("no predicate") } pv := reflect.ValueOf(predicate) pt := pv.Type() if pt.Kind() != reflect.Func || pt.NumIn() != 1 || pt.NumOut() != 1 || pt.In(0) != s.typ || pt.Out(0).Kind() != reflect.Bool { panic("wrong predicate signature, must be func(T) bool") } cnt := s.Len() cpy := s.makeEmptyCopy(cnt) for i := 0; i < cnt; i++ { v := s.slice.Index(i) ok := pv.Call([]reflect.Value{v})[0].Interface().(bool) if ok { cpy.slice = reflect.Append(cpy.slice, v) } } return cpy } // Distinct remove duplicate values func (s *serie) Distinct() Serie { cnt := s.Len() cpy := s.makeEmptyCopy(cnt) m := make(map[interface{}]bool) for i := 0; i < cnt; i++ { v := s.slice.Index(i) if _, ok := m[v.Interface()]; !ok { cpy.slice = reflect.Append(cpy.slice, v) m[v.Interface()] = true } } return cpy } // Pick picks some indexes {at} to create a new serie // If {at} is out of range, Pick will fill with a "zero" value func (s *serie) Pick(at ...int) Serie { cpy := s.makeEmptyCopy(len(at)) cnt := s.Len() for _, pos := range at { if pos >= 0 && pos < cnt { cpy.slice = reflect.Append(cpy.slice, s.slice.Index(pos)) } else { cpy.slice = reflect.Append(cpy.slice, s.converter.Call([]reflect.Value{reflect.Zero(s.typ)})...) } } return cpy } // Where to filter the serie on a predicate func (s *serie) Where(predicate func(interface{}) bool) Serie { cpy := s.makeEmptyCopy(s.Len()) if predicate == nil { return cpy } index := 0 for it := s.Iterator(); it.Next(); { v := it.Current() if predicate(v) { cpy.slice = reflect.Append(cpy.slice, s.slice.Index(index)) } index++ } return cpy } // NonNils selects all non-nils values in serie func (s *serie) NonNils() Serie { return s.Where(func(item interface{}) bool { return item != nil }) } ================================================ FILE: serie/select_test.go ================================================ package serie_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/datasweet/datatable/serie" ) func TestHead(t *testing.T) { s := serie.Int(1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s.Head(5), 1, 2, 3, 4, 5) assertSerieEq(t, s.Head(1), 1) assertSerieEq(t, s.Head(9), 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s.Head(10), 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s.Head(0)) assertSerieEq(t, s.Head(-1)) assertSerieEq(t, s.Head(5).Head(1), 1) } func TestTail(t *testing.T) { s := serie.Int(1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s.Tail(5), 5, 6, 7, 8, 9) assertSerieEq(t, s.Tail(1), 9) assertSerieEq(t, s.Tail(9), 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s.Tail(10)) assertSerieEq(t, s.Tail(0)) assertSerieEq(t, s.Tail(-1)) assertSerieEq(t, s.Tail(5).Tail(1), 9) } func TestSubset(t *testing.T) { s := serie.Int(1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s.Subset(4, 3), 5, 6, 7) assertSerieEq(t, s.Subset(7, 2), 8, 9) assertSerieEq(t, s.Subset(7, 3), 8, 9) assertSerieEq(t, s.Subset(8, 1), 9) assertSerieEq(t, s.Subset(0, 5), 1, 2, 3, 4, 5) assertSerieEq(t, s.Subset(0, 1), 1) assertSerieEq(t, s.Subset(5, 0)) assertSerieEq(t, s.Subset(5, -1)) assertSerieEq(t, s.Subset(-1, 5)) assertSerieEq(t, s.Subset(10, 5)) } /* func TestFilter(t *testing.T) { s := serie.Int(1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 1, 2, 3, 4, 5, 6, 7, 8, 9) assert.Panics(t, func() { s.Filter(nil) }) assert.Panics(t, func() { s.Filter(func(val float32) bool { return val == 3.14 }) }) res := s.Filter(func(val int) bool { return val%2 == 1 }) assertSerieEq(t, res, 1, 3, 5, 7, 9) } */ func TestDistinct(t *testing.T) { s := serie.Int( 31, 23, 98, 3, 59, 67, 5, 5, 87, 18, 3, 88, 7, 63, 29, 62, 37, 66, 87, 26, 24, 5, 62, 75, 69, 56, 15, 59, 40, 34, 68, 32, 34, 29, 90, 21, 8, 8, 100, 64, 30, 56, 73, 2, 65, 74, 3, 26, 92, 46, 6, 100, 35, 17, 91, 55, 99, 87, 9, 25, 55, 76, 39, 78, 43, 99, 35, 90, 36, 27, 52, 65, 33, 49, 84, 87, 42, 92, 27, 65, 48, 47, 74, 98, 76, 88, 18, 100, 69, 57, 69, 90, 74, 25, 64, 37, 63, 61, 85, 12, ) assertSerieEq(t, s.Distinct(), 31, 23, 98, 3, 59, 67, 5, 87, 18, 88, 7, 63, 29, 62, 37, 66, 26, 24, 75, 69, 56, 15, 40, 34, 68, 32, 90, 21, 8, 100, 64, 30, 73, 2, 65, 74, 92, 46, 6, 35, 17, 91, 55, 99, 9, 25, 76, 39, 78, 43, 36, 27, 52, 33, 49, 84, 42, 48, 47, 57, 61, 85, 12, ) } func TestPick(t *testing.T) { s := serie.Int(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s.Pick(4, 3), 4, 3) assertSerieEq(t, s.Pick(-1), 0) assertSerieEq(t, s.Pick(0, -1, 4, 3, 9, 10), 0, 0, 4, 3, 9, 0) s = serie.IntN(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s.Pick(4, 3), 4, 3) assertSerieEq(t, s.Pick(-1), nil) assertSerieEq(t, s.Pick(0, -1, 4, 3, 9, 10), 0, nil, 4, 3, 9, nil) } func TestWhere(t *testing.T) { s := serie.Int(1, 2, 3, 4, 5, 6, 7, 8, 9) assertSerieEq(t, s, 1, 2, 3, 4, 5, 6, 7, 8, 9) res := s.Where(nil) assert.NotNil(t, res) assert.Equal(t, 0, res.Len()) assert.Panics(t, func() { s.Where(func(val interface{}) bool { return val.(float32) == 3.14 }) }) res = s.Where(func(val interface{}) bool { return val.(int)%2 == 1 }) assertSerieEq(t, res, 1, 3, 5, 7, 9) } func TestNonNils(t *testing.T) { s := serie.IntN( 31, 23, 98, 3, 59, 67, 5, 5, 87, 18, 3, 88, 7, 63, 29, 62, 37, 66, 87, 26, 24, 5, 62, 75, 69, 56, 15, 59, 40, 34, 68, 32, 34, 29, 90, 21, 8, 8, 100, 64, 30, 56, 73, 2, 65, "74", 3, 26, 92, 46, 6, 100, 35, 17, 91, 55, 99, 87, 9, 25, 55, "teemo", 39, 78, 43, 99, 35, 90, 36, 27, 52, 65, 33, nil, 84, 87, 42, 92, 27, 65, 48, 47, 74, 98, 76, 88, 18, 100, 69, 57, 69, 90, 74, 25, 64, 37, 63, 61, 85, 12, ) assertSerieEq(t, s, 31, 23, 98, 3, 59, 67, 5, 5, 87, 18, 3, 88, 7, 63, 29, 62, 37, 66, 87, 26, 24, 5, 62, 75, 69, 56, 15, 59, 40, 34, 68, 32, 34, 29, 90, 21, 8, 8, 100, 64, 30, 56, 73, 2, 65, 74, 3, 26, 92, 46, 6, 100, 35, 17, 91, 55, 99, 87, 9, 25, 55, nil, 39, 78, 43, 99, 35, 90, 36, 27, 52, 65, 33, nil, 84, 87, 42, 92, 27, 65, 48, 47, 74, 98, 76, 88, 18, 100, 69, 57, 69, 90, 74, 25, 64, 37, 63, 61, 85, 12) assertSerieEq(t, s.NonNils(), 31, 23, 98, 3, 59, 67, 5, 5, 87, 18, 3, 88, 7, 63, 29, 62, 37, 66, 87, 26, 24, 5, 62, 75, 69, 56, 15, 59, 40, 34, 68, 32, 34, 29, 90, 21, 8, 8, 100, 64, 30, 56, 73, 2, 65, 74, 3, 26, 92, 46, 6, 100, 35, 17, 91, 55, 99, 87, 9, 25, 55, 39, 78, 43, 99, 35, 90, 36, 27, 52, 65, 33, 84, 87, 42, 92, 27, 65, 48, 47, 74, 98, 76, 88, 18, 100, 69, 57, 69, 90, 74, 25, 64, 37, 63, 61, 85, 12, ) } ================================================ FILE: serie/serie.go ================================================ package serie import ( "fmt" "reflect" "sort" ) type Serie interface { Type() reflect.Type Slice() interface{} // Underlying slice Get(at int) interface{} // T[i]. If T is an interfacer, returns Interfaced value All() []interface{} // Iterate Iterator() Iterator // Mutate Append(v ...interface{}) Prepend(v ...interface{}) error Insert(at int, v ...interface{}) error Set(at int, v interface{}) error Delete(at int) error Grow(size int) error Shrink(size int) error Concat(serie ...Serie) error Clear() // Select Head(size int) Serie Tail(size int) Serie Subset(at, size int) Serie Distinct() Serie Pick(at ...int) Serie Where(predicate func(interface{}) bool) Serie NonNils() Serie // Copy EmptyCopy() Serie Copy() Serie // Sort sort.Interface Compare(i, j int) int SortAsc() SortDesc() // // Print // Print(opts ...PrintOption) string // fmt.Stringer // Statistics Avg(opt ...StatOption) float64 Count(opt ...StatOption) int64 CountDistinct(opt ...StatOption) int64 Cusum(opt ...StatOption) []float64 Max(opt ...StatOption) float64 Min(opt ...StatOption) float64 Median(opt ...StatOption) float64 Stddev(opt ...StatOption) float64 Sum(opt ...StatOption) float64 Variance(opt ...StatOption) float64 } // Interfacer to convert a value of serie to interface{} // Used with serie.Get(at) serie.All() type Interfacer interface { Interface() interface{} } const ( Lt = -1 Eq = 0 Gt = 1 ) type serie struct { typ reflect.Type slice reflect.Value converter reflect.Value comparer reflect.Value interfacer bool } func New(typ interface{}, converter interface{}, comparer interface{}) Serie { if typ == nil { panic("arg 'typ' is not a concrete type") } if converter == nil { panic("nil converter") } if comparer == nil { panic("nil comparer") } rv := reflect.ValueOf(typ) kind := rv.Kind() if kind == reflect.Invalid { panic(fmt.Sprintf("type %T is invalid", rv)) } serie := &serie{} if kind == reflect.Slice { serie.slice = rv serie.typ = rv.Type().Elem() } else { serie.typ = rv.Type() serie.slice = reflect.MakeSlice(reflect.SliceOf(serie.typ), 0, 0) } // analyse converter convValue := reflect.ValueOf(converter) convType := convValue.Type() if convType.Kind() != reflect.Func || convType.NumIn() != 1 || convType.NumOut() != 1 || convType.In(0).Kind() != reflect.Interface || convType.Out(0) != serie.typ { panic(fmt.Sprintf("wrong converter signature, must be func(i interface{}) %s", serie.typ.Name())) } serie.converter = convValue // analyse comparer cmpValue := reflect.ValueOf(comparer) cmpType := cmpValue.Type() if cmpType.Kind() != reflect.Func || cmpType.NumIn() != 2 || cmpType.NumOut() != 1 || cmpType.In(0) != serie.typ || cmpType.In(1) != serie.typ || cmpType.Out(0).Kind() != reflect.Int { panic("wrong comparer signature, must be func(i, j T) int") } serie.comparer = cmpValue // analyse interfacer if serie.typ.Implements(reflect.TypeOf((*Interfacer)(nil)).Elem()) { serie.interfacer = true } return serie } // Len returns the len of the serie func (s *serie) Len() int { return s.slice.Len() } // Type returns the underlying type of serie func (s *serie) Type() reflect.Type { return s.typ } // Slice returns the underlying slice func (s *serie) Slice() interface{} { return s.slice.Interface() } // Get returns the value at index // If the serie is an interfacer, ie, values have custom Interface() func, // the Interface() func will be called. // So you can have difference between serie.Slice()[at] and serie.Get(at) func (s *serie) Get(at int) interface{} { if s.interfacer { return s.slice.Index(at).Interface().(Interfacer).Interface() } return s.slice.Index(at).Interface() } // All to get all values // Better to use serie.Iterator() if you want to work on values func (s *serie) All() []interface{} { all := make([]interface{}, 0, s.Len()) for it := s.Iterator(); it.Next(); { all = append(all, it.Current()) } return all } func (s *serie) String() string { return fmt.Sprintf("%+v", s.Slice()) } ================================================ FILE: serie/serie_bool.go ================================================ package serie import ( "github.com/datasweet/cast" ) func Bool(v ...interface{}) Serie { s := New(false, asBool, compareBool) if len(v) > 0 { s.Append(v...) } return s } func BoolN(v ...interface{}) Serie { s := New(NullBool{}, asNullBool, compareNullBool) if len(v) > 0 { s.Append(v...) } return s } func asBool(i interface{}) bool { b, _ := cast.AsBool(i) return b } func compareBool(a, b bool) int { if a == b { return Eq } if !a { return Lt } return Gt } type NullBool struct { Bool bool Valid bool } func (b NullBool) Interface() interface{} { if b.Valid { return b.Bool } return nil } func asNullBool(i interface{}) NullBool { var ni NullBool if i == nil { return ni } if v, ok := i.(NullBool); ok { return v } if v, ok := cast.AsBool(i); ok { ni.Bool = v ni.Valid = true } return ni } func compareNullBool(a, b NullBool) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } return compareBool(a.Bool, b.Bool) } ================================================ FILE: serie/serie_bool_test.go ================================================ package serie_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/datasweet/datatable/serie" ) func TestSerieBool(t *testing.T) { s := serie.Bool() assert.NotNil(t, s) s.Append(1, 0, true, "teemo", nil) assertSerieEq(t, s, true, false, true, false, false) s.SortAsc() assertSerieEq(t, s, false, false, false, true, true) s.SortDesc() assertSerieEq(t, s, true, true, false, false, false) } func TestSerieBoolN(t *testing.T) { s := serie.BoolN() assert.NotNil(t, s) s.Append(1, 0, true, "teemo", nil) assertSerieEq(t, s, true, false, true, nil, nil) s.SortAsc() assertSerieEq(t, s, nil, nil, false, true, true) s.SortDesc() assertSerieEq(t, s, true, true, false, nil, nil) } ================================================ FILE: serie/serie_float32.go ================================================ package serie import ( "github.com/datasweet/cast" ) func Float32(v ...interface{}) Serie { s := New(float32(0), asFloat32, compareFloat32) if len(v) > 0 { s.Append(v...) } return s } func Float32N(v ...interface{}) Serie { s := New(NullFloat32{}, asNullFloat32, compareNullFloat32) if len(v) > 0 { s.Append(v...) } return s } func asFloat32(i interface{}) float32 { f, _ := cast.AsFloat32(i) return f } func compareFloat32(a, b float32) int { if a == b { return Eq } if a < b { return Lt } return Gt } type NullFloat32 struct { Float32 float32 Valid bool } func (f NullFloat32) Interface() interface{} { if f.Valid { return f.Float32 } return nil } func asNullFloat32(i interface{}) NullFloat32 { var ni NullFloat32 if i == nil { return ni } if v, ok := i.(NullFloat32); ok { return v } if v, ok := cast.AsFloat32(i); ok { ni.Float32 = v ni.Valid = true } return ni } func compareNullFloat32(a, b NullFloat32) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } return compareFloat32(a.Float32, b.Float32) } ================================================ FILE: serie/serie_float32_test.go ================================================ package serie_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/datasweet/datatable/serie" ) func TestSerieFloat32(t *testing.T) { s := serie.Float32() assert.NotNil(t, s) s.Append(1, "23", 3.14, "teemo", true, nil) assertSerieEq(t, s, float32(1), float32(23), float32(3.14), float32(0), float32(1), float32(0)) s.SortAsc() assertSerieEq(t, s, float32(0), float32(0), float32(1), float32(1), float32(3.14), float32(23)) s.SortDesc() assertSerieEq(t, s, float32(23), float32(3.14), float32(1), float32(1), float32(0), float32(0)) } func TestSerieFloat32N(t *testing.T) { s := serie.Float32N() assert.NotNil(t, s) s.Append(1, "23", 3.14, "teemo", true, nil) assertSerieEq(t, s, float32(1), float32(23), float32(3.14), nil, float32(1), nil) s.SortAsc() assertSerieEq(t, s, nil, nil, float32(1), float32(1), float32(3.14), float32(23)) s.SortDesc() assertSerieEq(t, s, float32(23), float32(3.14), float32(1), float32(1), nil, nil) } ================================================ FILE: serie/serie_float64.go ================================================ package serie import ( "github.com/datasweet/cast" ) func Float64(v ...interface{}) Serie { s := New(float64(0), asFloat64, compareFloat64) if len(v) > 0 { s.Append(v...) } return s } func Float64N(v ...interface{}) Serie { s := New(NullFloat64{}, asNullFloat64, compareNullFloat64) if len(v) > 0 { s.Append(v...) } return s } func asFloat64(i interface{}) float64 { f, _ := cast.AsFloat64(i) return f } func compareFloat64(a, b float64) int { if a == b { return Eq } if a < b { return Lt } return Gt } type NullFloat64 struct { Float64 float64 Valid bool } func (f NullFloat64) Interface() interface{} { if f.Valid { return f.Float64 } return nil } func asNullFloat64(i interface{}) NullFloat64 { var ni NullFloat64 if i == nil { return ni } if v, ok := i.(NullFloat64); ok { return v } if v, ok := cast.AsFloat64(i); ok { ni.Float64 = v ni.Valid = true } return ni } func compareNullFloat64(a, b NullFloat64) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } return compareFloat64(a.Float64, b.Float64) } ================================================ FILE: serie/serie_float64_test.go ================================================ package serie_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/datasweet/datatable/serie" ) func TestSerieFloat64(t *testing.T) { s := serie.Float64() assert.NotNil(t, s) s.Append(1, "23", 3.14, "teemo", true, nil) assertSerieEq(t, s, float64(1), float64(23), float64(3.14), float64(0), float64(1), float64(0)) s.SortAsc() assertSerieEq(t, s, float64(0), float64(0), float64(1), float64(1), float64(3.14), float64(23)) s.SortDesc() assertSerieEq(t, s, float64(23), float64(3.14), float64(1), float64(1), float64(0), float64(0)) } func TestSerieFloat64N(t *testing.T) { s := serie.Float64N() assert.NotNil(t, s) s.Append(1, "23", 3.14, "teemo", true, nil) assertSerieEq(t, s, float64(1), float64(23), float64(3.14), nil, float64(1), nil) s.SortAsc() assertSerieEq(t, s, nil, nil, float64(1), float64(1), float64(3.14), float64(23)) s.SortDesc() assertSerieEq(t, s, float64(23), float64(3.14), float64(1), float64(1), nil, nil) } ================================================ FILE: serie/serie_int.go ================================================ package serie import ( "github.com/datasweet/cast" ) func Int(v ...interface{}) Serie { s := New(0, asInt, compareInt) if len(v) > 0 { s.Append(v...) } return s } func IntN(v ...interface{}) Serie { s := New(NullInt{}, asNullInt, compareNullInt) if len(v) > 0 { s.Append(v...) } return s } func asInt(i interface{}) int { n, _ := cast.AsInt(i) return n } func compareInt(a, b int) int { if a == b { return Eq } if a < b { return Lt } return Gt } type NullInt struct { Int int Valid bool } func (i NullInt) Interface() interface{} { if i.Valid { return i.Int } return nil } func asNullInt(i interface{}) NullInt { var ni NullInt if i == nil { return ni } if v, ok := i.(NullInt); ok { return v } if v, ok := cast.AsInt(i); ok { ni.Int = v ni.Valid = true } return ni } func compareNullInt(a, b NullInt) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } return compareInt(a.Int, b.Int) } ================================================ FILE: serie/serie_int32.go ================================================ package serie import ( "github.com/datasweet/cast" ) func Int32(v ...interface{}) Serie { s := New(int32(0), asInt32, compareInt32) if len(v) > 0 { s.Append(v...) } return s } func Int32N(v ...interface{}) Serie { s := New(NullInt32{}, asNullInt32, compareNullInt32) if len(v) > 0 { s.Append(v...) } return s } func asInt32(i interface{}) int32 { n, _ := cast.AsInt32(i) return n } func compareInt32(a, b int32) int { if a == b { return Eq } if a < b { return Lt } return Gt } type NullInt32 struct { Int32 int32 Valid bool } func (i NullInt32) Interface() interface{} { if i.Valid { return i.Int32 } return nil } func asNullInt32(i interface{}) NullInt32 { var ni NullInt32 if i == nil { return ni } if v, ok := i.(NullInt32); ok { return v } if v, ok := cast.AsInt32(i); ok { ni.Int32 = v ni.Valid = true } return ni } func compareNullInt32(a, b NullInt32) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } return compareInt32(a.Int32, b.Int32) } ================================================ FILE: serie/serie_int32_test.go ================================================ package serie_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/datasweet/datatable/serie" ) func TestSerieInt32(t *testing.T) { s := serie.Int32() assert.NotNil(t, s) s.Append(31, "23", 98.5, "teemo", true, -67, nil) assertSerieEq(t, s, int32(31), int32(23), int32(98), int32(0), int32(1), int32(-67), int32(0), ) s.SortAsc() assertSerieEq(t, s, int32(-67), int32(0), int32(0), int32(1), int32(23), int32(31), int32(98), ) s.SortDesc() assertSerieEq(t, s, int32(98), int32(31), int32(23), int32(1), int32(0), int32(0), int32(-67), ) } func TestSerieInt32N(t *testing.T) { s := serie.Int32N() assert.NotNil(t, s) s.Append(31, "23", 98.5, "teemo", true, -67, nil) assertSerieEq(t, s, int32(31), int32(23), int32(98), nil, int32(1), int32(-67), nil, ) s.SortAsc() assertSerieEq(t, s, nil, nil, int32(-67), int32(1), int32(23), int32(31), int32(98), ) s.SortDesc() assertSerieEq(t, s, int32(98), int32(31), int32(23), int32(1), int32(-67), nil, nil, ) } ================================================ FILE: serie/serie_int64.go ================================================ package serie import ( "github.com/datasweet/cast" ) func Int64(v ...interface{}) Serie { s := New(int64(0), asInt64, compareInt64) if len(v) > 0 { s.Append(v...) } return s } func Int64N(v ...interface{}) Serie { s := New(NullInt64{}, asNullInt64, compareNullInt64) if len(v) > 0 { s.Append(v...) } return s } func asInt64(i interface{}) int64 { n, _ := cast.AsInt64(i) return n } func compareInt64(a, b int64) int { if a == b { return Eq } if a < b { return Lt } return Gt } type NullInt64 struct { Int64 int64 Valid bool } func (i NullInt64) Interface() interface{} { if i.Valid { return i.Int64 } return nil } func asNullInt64(i interface{}) NullInt64 { var ni NullInt64 if i == nil { return ni } if v, ok := i.(NullInt64); ok { return v } if v, ok := cast.AsInt64(i); ok { ni.Int64 = v ni.Valid = true } return ni } func compareNullInt64(a, b NullInt64) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } return compareInt64(a.Int64, b.Int64) } ================================================ FILE: serie/serie_int64_test.go ================================================ package serie_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/datasweet/datatable/serie" ) func TestSerieInt64(t *testing.T) { s := serie.Int64() assert.NotNil(t, s) s.Append(31, "23", 98.5, "teemo", true, -67, nil) assertSerieEq(t, s, int64(31), int64(23), int64(98), int64(0), int64(1), int64(-67), int64(0), ) s.SortAsc() assertSerieEq(t, s, int64(-67), int64(0), int64(0), int64(1), int64(23), int64(31), int64(98), ) s.SortDesc() assertSerieEq(t, s, int64(98), int64(31), int64(23), int64(1), int64(0), int64(0), int64(-67), ) } func TestSerieInt64N(t *testing.T) { s := serie.Int64N() assert.NotNil(t, s) s.Append(31, "23", 98.5, "teemo", true, -67, nil) assertSerieEq(t, s, int64(31), int64(23), int64(98), nil, int64(1), int64(-67), nil, ) s.SortAsc() assertSerieEq(t, s, nil, nil, int64(-67), int64(1), int64(23), int64(31), int64(98), ) s.SortDesc() assertSerieEq(t, s, int64(98), int64(31), int64(23), int64(1), int64(-67), nil, nil, ) } ================================================ FILE: serie/serie_int_test.go ================================================ package serie_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/datasweet/datatable/serie" ) func TestSerieInt(t *testing.T) { s := serie.Int() assert.NotNil(t, s) s.Append(31, "23", 98.5, "teemo", true, -67, nil) assertSerieEq(t, s, 31, 23, 98, 0, 1, -67, 0) s.SortAsc() assertSerieEq(t, s, -67, 0, 0, 1, 23, 31, 98) s.SortDesc() assertSerieEq(t, s, 98, 31, 23, 1, 0, 0, -67) } func TestSerieIntN(t *testing.T) { s := serie.IntN() assert.NotNil(t, s) s.Append(31, "23", 98.5, "teemo", true, -67, nil) assertSerieEq(t, s, 31, 23, 98, nil, 1, -67, nil) s.SortAsc() assertSerieEq(t, s, nil, nil, -67, 1, 23, 31, 98) s.SortDesc() assertSerieEq(t, s, 98, 31, 23, 1, -67, nil, nil) } ================================================ FILE: serie/serie_raw.go ================================================ package serie import ( "fmt" "strings" ) func Raw(v ...interface{}) Serie { s := New(RawValue{}, asRawValue, compareRawValue) if len(v) > 0 { s.Append(v...) } return s } type RawValue struct { Value interface{} Valid bool } func (r RawValue) Interface() interface{} { if r.Valid { return r.Value } return nil } func (r RawValue) String() string { return fmt.Sprint(r.Value) } func asRawValue(i interface{}) RawValue { if rv, ok := i.(RawValue); ok { return rv } var r RawValue if i == nil { return r } r.Valid = true r.Value = i return r } func compareRawValue(a, b RawValue) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } return strings.Compare(a.String(), b.String()) } ================================================ FILE: serie/serie_raw_test.go ================================================ package serie_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/datasweet/datatable/serie" ) func TestSerieRaw(t *testing.T) { s := serie.Raw() assert.NotNil(t, s) s.Append(31, "23", 98.5, "teemo", true, nil, -67) assertSerieEq(t, s, 31, "23", 98.5, "teemo", true, nil, -67) s.SortAsc() assertSerieEq(t, s, nil, -67, "23", 31, 98.5, "teemo", true) s.SortDesc() assertSerieEq(t, s, true, "teemo", 98.5, 31, "23", -67, nil) } ================================================ FILE: serie/serie_string.go ================================================ package serie import ( "strings" "github.com/datasweet/cast" ) // String to create a new string serie func String(v ...interface{}) Serie { s := New("", asString, strings.Compare) if len(v) > 0 { s.Append(v...) } return s } // StringN to create a new serie with null value handling func StringN(v ...interface{}) Serie { s := New(NullString{}, asNullString, compareNullString) if len(v) > 0 { s.Append(v...) } return s } func asString(i interface{}) string { s, _ := cast.AsString(i) return s } type NullString struct { String string Valid bool } func (s NullString) Interface() interface{} { if s.Valid { return s.String } return nil } func asNullString(i interface{}) NullString { var ns NullString if i == nil { return ns } if v, ok := i.(NullString); ok { return v } if v, ok := cast.AsString(i); ok { ns.String = v ns.Valid = true } return ns } func compareNullString(a, b NullString) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } return strings.Compare(a.String, b.String) } ================================================ FILE: serie/serie_string_test.go ================================================ package serie_test import ( "testing" "github.com/datasweet/datatable/serie" ) func TestString(t *testing.T) { s := serie.String("A00103", 1, 3.14, true, nil) assertSerieEq(t, s, "A00103", "1", "3.14", "true", "") } ================================================ FILE: serie/serie_test.go ================================================ package serie_test import ( "testing" "github.com/datasweet/cast" "github.com/datasweet/datatable/serie" "github.com/stretchr/testify/assert" ) func TestNewSerie(t *testing.T) { assert.Panics(t, func() { serie.New(nil, nil, nil) }) assert.Panics(t, func() { serie.New(1, nil, nil) }) assert.Panics(t, func() { serie.New(1, func(i interface{}) float32 { f, _ := cast.AsFloat32(i) return f }, func(i, j int) int { return serie.Eq }) }) assert.Panics(t, func() { serie.New(1, func(i interface{}) int { f, _ := cast.AsInt(i) return f }, nil) }) // OK s := serie.New(1, func(i interface{}) int { f, _ := cast.AsInt(i) return f }, func(i, j int) int { return serie.Eq }) assert.NotNil(t, s) s.Append(1, 2, 3, 4, 5) assertSerieEq(t, s, 1, 2, 3, 4, 5) } ================================================ FILE: serie/serie_time.go ================================================ package serie import ( "time" "github.com/datasweet/cast" ) // Time to create a time serie func Time(format ...string) Serie { return New(time.Time{}, asTime(format), compareTime) } // TimeN to create a time serie with nil value func TimeN(format ...string) Serie { return New(NullTime{}, asNullTime(format), compareNullTime) } func compareTime(a, b time.Time) int { if a.Equal(b) { return Eq } if a.Before(b) { return Lt } return Gt } func asTime(formats []string) func(interface{}) time.Time { return func(i interface{}) time.Time { t, _ := cast.AsTime(i, formats...) return t } } type NullTime struct { Time time.Time Valid bool } func (t NullTime) Interface() interface{} { if t.Valid { return t.Time } return nil } func asNullTime(formats []string) func(interface{}) NullTime { return func(i interface{}) NullTime { var ni NullTime if i == nil { return ni } if v, ok := i.(NullTime); ok { return v } if v, ok := cast.AsTime(i, formats...); ok { ni.Time = v ni.Valid = true } return ni } } func compareNullTime(a, b NullTime) int { if !b.Valid { if !a.Valid { return Eq } return Gt } if !a.Valid { return Lt } return compareTime(a.Time, b.Time) } ================================================ FILE: serie/serie_time_test.go ================================================ package serie_test import ( "testing" "time" "github.com/datasweet/datatable/serie" ) func TestSerieTime(t *testing.T) { s := serie.Time() s.Append(1551435220270, "2019-03-01", "2019-03-01 10:13:40", "2019-03-01T10:13:40.27Z", "01/03/2019", "01/03/2019 10:13:40", "wrong") date := time.Date(2019, time.March, 1, 0, 0, 0, 0, time.UTC) // only date datetime := time.Date(2019, time.March, 1, 10, 13, 40, 0, time.UTC) // date + time timestamp := time.Unix(0, 1551435220270*int64(time.Millisecond)).UTC() // date + time + ns assertSerieEq(t, s, timestamp, date, datetime, timestamp, date, datetime, time.Time{}) s = serie.Time("02/01/06", "02/01/06 15:04:05") s.Append("01/03/19", "01/03/19 10:13:40", "wrong") assertSerieEq(t, s, date, datetime, time.Time{}) } func TestSerieTimeN(t *testing.T) { s := serie.TimeN() s.Append(1551435220270, "2019-03-01", "2019-03-01 10:13:40", "2019-03-01T10:13:40.27Z", "01/03/2019", "01/03/2019 10:13:40", "wrong") date := time.Date(2019, time.March, 1, 0, 0, 0, 0, time.UTC) // only date datetime := time.Date(2019, time.March, 1, 10, 13, 40, 0, time.UTC) // date + time timestamp := time.Unix(0, 1551435220270*int64(time.Millisecond)).UTC() // date + time + ns assertSerieEq(t, s, timestamp, date, datetime, timestamp, date, datetime, nil) s = serie.TimeN("02/01/06", "02/01/06 15:04:05") s.Append("01/03/19", "01/03/19 10:13:40", "wrong") assertSerieEq(t, s, date, datetime, nil) } ================================================ FILE: serie/sort.go ================================================ package serie import ( "reflect" "sort" ) func (s *serie) Swap(i, j int) { tmp := reflect.New(s.typ).Elem() a, b := s.slice.Index(i), s.slice.Index(j) tmp.Set(a) a.Set(b) b.Set(tmp) } func (s *serie) Less(i, j int) bool { return s.Compare(i, j) == Lt } // Compare values at indexes i, j // panic if out of range func (s *serie) Compare(i, j int) int { return s.comparer.Call([]reflect.Value{ s.slice.Index(i), s.slice.Index(j), })[0].Interface().(int) } func (s *serie) SortAsc() { sort.Sort(s) } func (s *serie) SortDesc() { sort.Sort(sort.Reverse(s)) } ================================================ FILE: serie/sort_test.go ================================================ package serie_test import ( "sort" "testing" "github.com/datasweet/datatable/serie" ) func TestSortInt(t *testing.T) { random := []interface{}{ 31, 23, 98, 3, 59, 67, 5, 5, 87, 18, 3, 88, 7, 63, 29, 62, 37, 66, 87, 26, 24, 5, 62, 75, 69, 56, 15, 59, 40, 34, 68, 32, 34, 29, 90, 21, 8, 8, 100, 64, 30, 56, 73, 2, 65, 74, 3, 26, 92, 46, 6, 100, 35, 17, 91, 55, 99, 87, 9, 25, 55, 76, 39, 78, 43, 99, 35, 90, 36, 27, 52, 65, 33, 49, 84, 87, 42, 92, 27, 65, 48, 47, 74, 98, 76, 88, 18, 100, 69, 57, 69, 90, 74, 25, 64, 37, 63, 61, 85, 12, } s := serie.Int(random...) sort.Slice(random, func(i, j int) bool { return random[i].(int) < random[j].(int) }) s.SortAsc() assertSerieEq(t, s, random...) sort.Slice(random, func(i, j int) bool { return random[i].(int) > random[j].(int) }) s.SortDesc() assertSerieEq(t, s, random...) } func TestSortString(t *testing.T) { s := serie.String("A00103", "A00105", "A00104", "A00106", "A00104", nil) s.SortAsc() assertSerieEq(t, s, "", "A00103", "A00104", "A00104", "A00105", "A00106") s.SortDesc() assertSerieEq(t, s, "A00106", "A00105", "A00104", "A00104", "A00103", "") } ================================================ FILE: serie/stat.go ================================================ package serie import ( "math" "sort" "gonum.org/v1/gonum/floats" "gonum.org/v1/gonum/stat" ) // An aggregate function performs a calculation on a set of values, and returns a single value. // Aggregate functions ignore null values. type StatOptions struct { Missing *float64 // replaces missing values with a value } type StatOption func(opts *StatOptions) // Missing to treats all missing values (ie no-nils) as a value func Missing(f float64) StatOption { return func(opts *StatOptions) { opts.Missing = &f } } func (s *serie) asFloats(opt ...StatOption) []float64 { var options StatOptions for _, o := range opt { o(&options) } conv := AsFloat64(s, options.Missing) return conv.Slice().([]float64) } // Avg returns the average of non-nil values // returns NaN if no value func (s *serie) Avg(opt ...StatOption) float64 { src := s.asFloats(opt...) if len(src) == 0 { return math.NaN() } return stat.Mean(src, nil) } // Count returns the number of non-nil values func (s *serie) Count(opt ...StatOption) int64 { src := s.NonNils() return int64(src.Len()) } // CountDistinct returns the number of unique non-nil values func (s *serie) CountDistinct(opt ...StatOption) int64 { src := s.NonNils().Distinct() return int64(src.Len()) } // Cusum returns the cumulative sum of non-nil values // returns NaN if no value func (s *serie) Cusum(opt ...StatOption) []float64 { opts := make([]StatOption, 0, len(opt)+1) opts = append(opts, Missing(0)) opts = append(opts, opt...) src := s.asFloats(opts...) if len(src) == 0 { return src } dst := make([]float64, 0, len(src)) floats.CumSum(dst, src) return dst } // Max returns the maximum of non-nil values // returns NaN if no value func (s *serie) Max(opt ...StatOption) float64 { src := s.asFloats(opt...) if len(src) == 0 { return math.NaN() } return floats.Max(src) } // Min returns the minimum of non-nil values // returns NaN if no value func (s *serie) Min(opt ...StatOption) float64 { src := s.asFloats(opt...) if len(src) == 0 { return math.NaN() } return floats.Min(src) } // Median returns the median value of non-nil values // returns NaN if no value func (s *serie) Median(opt ...StatOption) float64 { src := s.asFloats(opt...) if len(src) == 0 { return math.NaN() } // stat.Quantile needs the input slice to be sorted. sort.Float64s(src) // computes the median of the dataset. return stat.Quantile(0.5, stat.Empirical, src, nil) } // Stddev returns the standard deviation of non-nils values // returns NaN if no value func (s *serie) Stddev(opt ...StatOption) float64 { src := s.asFloats(opt...) if len(src) == 0 { return math.NaN() } return stat.StdDev(src, nil) } // Sum returns the sum of non-nil values func (s *serie) Sum(opt ...StatOption) float64 { src := s.asFloats(opt...) if len(src) == 0 { return 0 } return floats.Sum(src) } // Variance returns the variance of non-nil values // returns NaN if no value func (s *serie) Variance(opt ...StatOption) float64 { src := s.asFloats(opt...) if len(src) == 0 { return math.NaN() } return stat.Variance(src, nil) } ================================================ FILE: serie/stat_test.go ================================================ package serie_test import ( "fmt" "math" "sort" "testing" "github.com/datasweet/datatable/serie" "github.com/stretchr/testify/assert" "gonum.org/v1/gonum/stat" ) func TestAvg(t *testing.T) { xs := []float64{ 32.32, 56.98, 21.52, 44.32, 55.63, 13.75, 43.47, 43.34, 12.34, } s := serie.Float64(xs) assert.NotNil(t, s) assert.Equal(t, 9, s.Len()) assert.Equal(t, stat.Mean(xs, nil), s.Avg()) s = serie.Float64N(xs, "teemo", "nil") assert.NotNil(t, s) assert.Equal(t, 11, s.Len()) assert.Equal(t, stat.Mean(xs, nil), s.Avg()) assert.Greater(t, stat.Mean(xs, nil), s.Avg(serie.Missing(0))) } func TestCount(t *testing.T) { xs := []float64{ 32.32, 56.98, 21.52, 44.32, 55.63, 13.75, 43.47, 43.34, 12.34, } s := serie.Float64(xs) assert.NotNil(t, s) assert.Equal(t, 9, s.Len()) assert.Equal(t, int64(9), s.Count()) s = serie.Float64N(xs, "teemo", "nil") assert.NotNil(t, s) assert.Equal(t, 11, s.Len()) assert.Equal(t, int64(9), s.Count()) } func TestSum(t *testing.T) { s := serie.Float64N(1, "23", 3.14, "teemo", true, nil) assert.NotNil(t, s) assertSerieEq(t, s, float64(1), float64(23), float64(3.14), nil, float64(1), nil) assert.Equal(t, 28.14, s.Sum()) } func TestMedian(t *testing.T) { xs := []float64{ 32.32, 56.98, 21.52, 44.32, 55.63, 13.75, 43.47, 43.34, 12.34, } fmt.Printf("data: %v\n", xs) // computes the weighted mean of the dataset. // we don't have any weights (ie: all weights are 1) // so we just pass a nil slice. mean := stat.Mean(xs, nil) variance := stat.Variance(xs, nil) stddev := math.Sqrt(variance) // stat.Quantile needs the input slice to be sorted. sort.Float64s(xs) fmt.Printf("data: %v (sorted)\n", xs) // computes the median of the dataset. // here as well, we pass a nil slice as weights. median := stat.Quantile(0.5, stat.Empirical, xs, nil) fmt.Printf("mean= %v\n", mean) fmt.Printf("median= %v\n", median) fmt.Printf("variance= %v\n", variance) fmt.Printf("std-dev= %v\n", stddev) } ================================================ FILE: serie/utils_test.go ================================================ package serie_test import ( "testing" "github.com/stretchr/testify/assert" "github.com/datasweet/datatable/serie" ) func assertSerieEq(t *testing.T, s serie.Serie, v ...interface{}) { assert.NotNil(t, s) assert.Equal(t, len(v), s.Len()) index := 0 for it := s.Iterator(); it.Next(); { assert.Equalf(t, v[index], it.Current(), "At index %d", index) index++ } } ================================================ FILE: serie_test.go ================================================ package datatable import ( "testing" "github.com/stretchr/testify/assert" ) func TestSerieFactory(t *testing.T) { typs := ColumnTypes() for _, typ := range typs { assert.NotPanics(t, func() { newColumnSerie(typ, ColumnOptions{}) }, typ) } } ================================================ FILE: sort.go ================================================ package datatable import ( "sort" "github.com/datasweet/datatable/serie" ) // SortBy defines a sort to be applied type SortBy struct { Column string Desc bool index int } // credits : https://stackoverflow.com/questions/36122668/how-to-sort-struct-with-multiple-sort-parameters type sorter struct { t *DataTable by []SortBy } func (s *sorter) Len() int { return s.t.nrows } func (s *sorter) Swap(i, j int) { s.t.SwapRow(i, j) } func (s *sorter) Less(i, j int) bool { for _, by := range s.by { sr := s.t.cols[by.index].serie switch cmp := sr.Compare(i, j); cmp { case serie.Eq: continue case serie.Gt: return by.Desc case serie.Lt: return !by.Desc } } return false } // Sort the table func (t *DataTable) Sort(by ...SortBy) *DataTable { cpy := t.Copy() if len(by) == 0 { return cpy } for i := range by { b := &by[i] // Check if column exists b.index = t.ColumnIndex(b.Column) if b.index < 0 { return cpy } } srt := &sorter{ t: cpy, by: by, } sort.Sort(srt) return cpy } ================================================ FILE: sort_test.go ================================================ package datatable_test import ( "testing" "time" "github.com/datasweet/datatable" "github.com/stretchr/testify/assert" ) func TestSort(t *testing.T) { // from join test customers, orders := sampleForJoin() dt, err := customers.LeftJoin(orders, datatable.On("[Customers].[id]", "[Orders].[user_id]")) assert.NoError(t, err) assert.NotNil(t, dt) checkTable(t, dt, "id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, 4, "Luc", "Rolland", "lucrolland@example.com", "Marseille", nil, nil, nil, ) sorted := dt.Sort(datatable.SortBy{Column: "num_facture", Desc: true}) checkTable(t, sorted, "id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 4, "Luc", "Rolland", "lucrolland@example.com", "Marseille", nil, nil, nil, ) // dt must not be modified sorted = dt.Sort(datatable.SortBy{Column: "ville"}, datatable.SortBy{Column: "id", Desc: true}) checkTable(t, sorted, "id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 4, "Luc", "Rolland", "lucrolland@example.com", "Marseille", nil, nil, nil, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, ) sorted = dt.Sort(datatable.SortBy{Column: "ville"}, datatable.SortBy{Column: "prix_total", Desc: true}) checkTable(t, sorted, "id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 4, "Luc", "Rolland", "lucrolland@example.com", "Marseille", nil, nil, nil, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, ) } ================================================ FILE: spec.md ================================================ ## Que doit faire notre table - input directement depuis json => traiter les NaN, les json dates, etc. - input directement depuis un csv - faire des calculs (expr) - formatter la table avec les options: précision, numeral, etc. - types de colonnes : - int - uint - bool - float - decimal - string - time - serie.... [ N°commande, Produit, Prix ] A, toto, 10 A, tata, 15 A, titi , 347 B, lionel, 3568 [N° commande, Produits] A, [{nom, prix}, {nom, prix}] => flatten(table) A, toto, 10 A, tata, 15 A, titi , 347 B, lionel, 3568 ================================================ FILE: table.go ================================================ package datatable import ( "fmt" "strings" ) // New creates a new datatable func New(name string) *DataTable { return &DataTable{name: name} } // DataTable is our main struct type DataTable struct { name string cols []*column nrows int dirty bool hasExpr bool } // Name returns the datatable's name func (t *DataTable) Name() string { return t.name } // Rename the datatable func (t *DataTable) Rename(name string) { t.name = name } // NumRows returns the number of rows in datatable func (t *DataTable) NumRows() int { return t.nrows } // NumCols returns the number of visible columns in datatable func (t *DataTable) NumCols() int { return len(t.Columns()) } // Columns returns the visible column names in datatable func (t *DataTable) Columns() []string { var cols []string for _, col := range t.cols { if col.IsVisible() { cols = append(cols, col.Name()) } } return cols } // HiddenColumns returns the hidden column names in datatable func (t *DataTable) HiddenColumns() []string { var cols []string for _, col := range t.cols { if !col.IsVisible() { cols = append(cols, col.Name()) } } return cols } // Column gets the column with name // returns nil if not found func (t *DataTable) Column(name string) Column { for _, col := range t.cols { if col.Name() == name { return col } } return nil } // ColumnIndex gets the index of the column with name // returns -1 if not found func (t *DataTable) ColumnIndex(name string) int { for i, col := range t.cols { if col.Name() == name { return i } } return -1 } // Records returns the rows in datatable as string // Computes all expressions. func (t *DataTable) Records() [][]string { if t == nil { return nil } if err := t.evaluateExpressions(); err != nil { panic(err) } // visible columns var cols []int for i, col := range t.cols { if col.IsVisible() { cols = append(cols, i) } } rows := make([][]string, 0, t.nrows) for i := 0; i < t.nrows; i++ { r := make([]string, 0, len(cols)) for _, pos := range cols { r = append(r, fmt.Sprintf("%v", t.cols[pos].serie.Get(i))) } rows = append(rows, r) } return rows } // Rows returns the rows in datatable // Computes all expressions. func (t *DataTable) Rows(opt ...ExportOption) []Row { if t == nil { return nil } opts := newExportOptions(opt...) if err := t.evaluateExpressions(); err != nil { panic(err) } // visible columns cols := make(map[string]int) for i, col := range t.cols { if opts.WithHiddenCols || col.IsVisible() { cols[col.Name()] = i } } rows := make([]Row, 0, t.nrows) for i := 0; i < t.nrows; i++ { r := make(Row, len(cols)) for name, pos := range cols { r[name] = t.cols[pos].serie.Get(i) } rows = append(rows, r) } return rows } func (t *DataTable) String() string { var sb strings.Builder t.Print(&sb) return sb.String() } // Row gets the row at index func (t *DataTable) Row(at int, opt ...ExportOption) Row { opts := newExportOptions(opt...) t.evaluateExpressions() r := make(Row, len(t.cols)) for _, col := range t.cols { if opts.WithHiddenCols || col.IsVisible() { r[col.name] = col.serie.Get(at) } } return r } ================================================ FILE: table_print.go ================================================ package datatable import ( "fmt" "io" "os" "strings" "github.com/olekukonko/tablewriter" ) // PrintOptions to control the printer type PrintOptions struct { ColumnName bool ColumnType bool RowNumber bool MaxRows int } type PrintOption func(opts *PrintOptions) func PrintColumnName(v bool) PrintOption { return func(opts *PrintOptions) { opts.ColumnName = v } } func PrintColumnType(v bool) PrintOption { return func(opts *PrintOptions) { opts.ColumnType = v } } func PrintRowNumber(v bool) PrintOption { return func(opts *PrintOptions) { opts.RowNumber = v } } func PrintMaxRows(v int) PrintOption { return func(opts *PrintOptions) { opts.MaxRows = v } } // Print the tables with options func (t *DataTable) Print(writer io.Writer, opt ...PrintOption) { options := PrintOptions{ ColumnName: true, ColumnType: true, RowNumber: true, MaxRows: 100, } for _, o := range opt { o(&options) } if writer == nil { writer = os.Stdout } tw := tablewriter.NewWriter(writer) tw.SetAutoWrapText(false) tw.SetHeaderAlignment(tablewriter.ALIGN_LEFT) tw.SetAlignment(tablewriter.ALIGN_LEFT) tw.SetCenterSeparator("") tw.SetColumnSeparator("") tw.SetRowSeparator("") tw.SetHeaderLine(false) tw.SetBorder(false) tw.SetTablePadding("\t") tw.SetNoWhiteSpace(true) if options.ColumnName || options.ColumnType { headers := make([]string, 0, len(t.cols)) for _, col := range t.cols { if !col.IsVisible() { continue } var h []string if options.ColumnName { h = append(h, col.Name()) } if options.ColumnType { h = append(h, fmt.Sprintf("<%s>", col.serie.Type().Name())) } headers = append(headers, strings.Join(h, " ")) } tw.SetHeader(headers) } if options.MaxRows > 1 && options.MaxRows <= t.NumRows() { mr := options.MaxRows / 2 tw.AppendBulk(t.Head(mr).Records()) seps := make([]string, 0, len(t.cols)) for _, col := range t.cols { if !col.IsVisible() { continue } seps = append(seps, "...") } tw.Append(seps) tw.AppendBulk(t.Tail(mr).Records()) } else { tw.AppendBulk(t.Records()) } tw.Render() } ================================================ FILE: table_print_test.go ================================================ package datatable_test import ( "os" "testing" "github.com/datasweet/datatable" ) func TestPrint(t *testing.T) { tb := New(t) checkTable(t, tb, "champ", "champion", "win", "loose", "winRate", "sum", "ok", "Malzahar", "MALZAHAR", 10, 6, "62.5 %", 696.0, true, "Xerath", "XERATH", 20, 5, "80 %", 696.0, true, "Teemo", "TEEMO", 666, 666, "50 %", 696.0, true, ) tb.Print(os.Stdout, datatable.PrintColumnType(false), datatable.PrintMaxRows(3)) } ================================================ FILE: table_test.go ================================================ package datatable_test import ( "fmt" "testing" "github.com/datasweet/datatable" "github.com/stretchr/testify/assert" ) func TestNewTable(t *testing.T) { tb := datatable.New("test") assert.Equal(t, 0, tb.NumCols()) assert.NoError(t, tb.AddColumn("sessions", datatable.Int, datatable.Values(120))) assert.NoError(t, tb.AddColumn("bounces", datatable.Int)) assert.NoError(t, tb.AddColumn("bounceRate", datatable.Float64)) assert.Error(t, tb.AddColumn("bounces", datatable.Int, datatable.Values(11))) assert.Error(t, tb.AddColumn(" ", datatable.Int, datatable.Values(11))) assert.Error(t, tb.AddColumn("nil", datatable.ColumnType("unknown"))) assert.NoError(t, tb.AddColumn("hidden", datatable.Int, datatable.Values(34), datatable.ColumnHidden(true))) assert.Equal(t, []string{"sessions", "bounces", "bounceRate"}, tb.Columns()) assert.Equal(t, 1, tb.NumRows()) assert.NoError(t, tb.AddColumn("pageViews", datatable.Int, datatable.Values(1, 2, 3, 4, 5))) assert.Equal(t, 4, tb.NumCols()) assert.Equal(t, 5, tb.NumRows()) fmt.Println(tb) checkTable(t, tb, "sessions", "bounces", "bounceRate", "pageViews", 120, nil, nil, 1, nil, nil, nil, 2, nil, nil, nil, 3, nil, nil, nil, 4, nil, nil, nil, 5, ) } func TestNewRow(t *testing.T) { tb := datatable.New("test") assert.NoError(t, tb.AddColumn("champ", datatable.String)) assert.Equal(t, 1, tb.NumCols()) assert.Equal(t, 0, tb.NumRows()) r := make(datatable.Row) r["champ"] = "Malzahar" tb.Append(r) assert.Equal(t, 1, tb.NumRows()) tb.Append(nil) assert.Equal(t, 1, tb.NumRows()) tb.Append() assert.Equal(t, 1, tb.NumRows()) tb.Append( tb.NewRow().Set("champ", "Xerath"), tb.NewRow().Set("satan", "Teemo"), // wrong column => not set tb.NewRow().Set("champ", "Ahri"), ) checkTable(t, tb, "champ", "Malzahar", "Xerath", nil, "Ahri", ) tb.AddColumn("win", datatable.Int) checkTable(t, tb, "champ", "win", "Malzahar", nil, "Xerath", nil, nil, nil, "Ahri", nil, ) tb.AddColumn("loose", datatable.Int, datatable.Values(3, 4, nil)) checkTable(t, tb, "champ", "win", "loose", "Malzahar", nil, 3, "Xerath", nil, 4, nil, nil, nil, "Ahri", nil, nil, ) } func TestExprColumn(t *testing.T) { tb := datatable.New("test") tb.AddColumn("champ", datatable.String, datatable.Values("Malzahar", "Xerath", "Teemo")) tb.AddColumn("champion", datatable.String, datatable.Expr("upper(`champ`)")) tb.AddColumn("win", datatable.Int, datatable.Values(10, 20, 666)) tb.AddColumn("loose", datatable.Int, datatable.Values(6, 5, 666)) tb.AddColumn("winRate", datatable.String, datatable.Expr("(`win` * 100 / (`win` + `loose`)) ~ \" %\"")) tb.AddColumn("sum", datatable.Int, datatable.Expr("sum(`win`)")) tb.AddColumn("ok", datatable.Bool, datatable.Expr("true")) checkTable(t, tb, "champ", "champion", "win", "loose", "winRate", "sum", "ok", "Malzahar", "MALZAHAR", 10, 6, "62.5 %", 696, true, "Xerath", "XERATH", 20, 5, "80 %", 696, true, "Teemo", "TEEMO", 666, 666, "50 %", 696, true, ) } func TestAppendRow(t *testing.T) { tb := datatable.New("test") assert.NoError(t, tb.AddColumn("champ", datatable.String)) assert.NoError(t, tb.AddColumn("win", datatable.Int)) assert.NoError(t, tb.AddColumn("loose", datatable.Int)) assert.NoError(t, tb.AddColumn("winRate", datatable.Float64, datatable.Expr("(`win` * 100 / (`win` + `loose`))"))) assert.Error(t, tb.AddColumn("winRate", datatable.String, datatable.Expr("test"))) assert.NoError(t, tb.AppendRow("Xerath", 25, 15, "expr")) assert.NoError(t, tb.AppendRow("Malzahar", 16, 16, nil)) assert.NoError(t, tb.AppendRow("Vel'Koz", 7, 5, 3)) checkTable(t, tb, "champ", "win", "loose", "winRate", "Xerath", 25, 15, 62.5, "Malzahar", 16, 16, 50.0, "Vel'Koz", 7, 5, 58.333333333333336, ) } func TestRows(t *testing.T) { tb := datatable.New("test") assert.NoError(t, tb.AddColumn("champ", datatable.String)) assert.NoError(t, tb.AddColumn("win", datatable.Int)) assert.NoError(t, tb.AddColumn("loose", datatable.Int, datatable.ColumnHidden(true))) assert.NoError(t, tb.AddColumn("winRate", datatable.Float64, datatable.Expr("(`win` * 100 / (`win` + `loose`))"))) assert.Error(t, tb.AddColumn("winRate", datatable.String, datatable.Expr("test"))) assert.NoError(t, tb.AppendRow("Xerath", 25, 15, "expr")) assert.NoError(t, tb.AppendRow("Malzahar", 16, 16, nil)) assert.NoError(t, tb.AppendRow("Vel'Koz", 7, 5, 3)) checkTable(t, tb, "champ", "win", "winRate", "Xerath", 25, 62.5, "Malzahar", 16, 50.0, "Vel'Koz", 7, 58.333333333333336, ) for _, r := range tb.Rows() { assert.Len(t, r, 3) } for _, r := range tb.Rows(datatable.ExportHidden(true)) { assert.Len(t, r, 4) } } func TestRow(t *testing.T) { tb := datatable.New("test") assert.NoError(t, tb.AddColumn("champ", datatable.String)) assert.NoError(t, tb.AddColumn("win", datatable.Int)) assert.NoError(t, tb.AddColumn("loose", datatable.Int, datatable.ColumnHidden(true))) assert.NoError(t, tb.AddColumn("winRate", datatable.Float64, datatable.Expr("(`win` * 100 / (`win` + `loose`))"))) assert.Error(t, tb.AddColumn("winRate", datatable.String, datatable.Expr("test"))) assert.NoError(t, tb.AppendRow("Xerath", 25, 15, "expr")) assert.NoError(t, tb.AppendRow("Malzahar", 16, 16, nil)) assert.NoError(t, tb.AppendRow("Vel'Koz", 7, 5, 3)) checkTable(t, tb, "champ", "win", "winRate", "Xerath", 25, 62.5, "Malzahar", 16, 50.0, "Vel'Koz", 7, 58.333333333333336, ) r := tb.Row(0) assert.Len(t, r, 3) assert.Equal(t, r.Get("champ"), "Xerath") assert.Equal(t, r.Get("win"), 25) assert.Equal(t, r.Get("winRate"), 62.5) assert.Nil(t, r.Get("loose")) r = tb.Row(0, datatable.ExportHidden(true)) assert.Len(t, r, 4) assert.Equal(t, r.Get("champ"), "Xerath") assert.Equal(t, r.Get("win"), 25) assert.Equal(t, r.Get("winRate"), 62.5) assert.Equal(t, r.Get("loose"), 15) } ================================================ FILE: test/main.go ================================================ package main import ( "fmt" "log" "os" "time" "github.com/datasweet/datatable" "github.com/datasweet/datatable/import/csv" ) func main() { dt, err := csv.Import("csv", "phone_data.csv", csv.HasHeader(true), csv.AcceptDate("02/01/06 15:04"), csv.AcceptDate("2006-01"), ) if err != nil { log.Fatalf("reading csv: %v", err) } dt.Print(os.Stdout, datatable.PrintMaxRows(24)) dt2, err := dt.Aggregate(datatable.AggregateBy{Type: datatable.Count, Field: "index"}) if err != nil { log.Fatalf("aggregate COUNT('index'): %v", err) } fmt.Println(dt2) groups, err := dt.GroupBy(datatable.GroupBy{ Name: "year", Type: datatable.Int, Keyer: func(row datatable.Row) (interface{}, bool) { if d, ok := row["date"]; ok { if tm, ok := d.(time.Time); ok { return tm.Year(), true } } return nil, false }, }) if err != nil { log.Fatalf("GROUP BY 'year': %v", err) } dt3, err := groups.Aggregate( datatable.AggregateBy{Type: datatable.Sum, Field: "duration"}, datatable.AggregateBy{Type: datatable.CountDistinct, Field: "network"}, ) if err != nil { log.Fatalf("Aggregate SUM('duration'), COUNT_DISTINCT('network') GROUP BY 'year': %v", err) } fmt.Println(dt3) } ================================================ FILE: test/phone_data.csv ================================================ index,date,duration,item,month,network,network_type 0,15/10/14 06:58,34.429,data,2014-11,data,data 1,15/10/14 06:58,13,call,2014-11,Vodafone,mobile 2,15/10/14 14:46,23,call,2014-11,Meteor,mobile 3,15/10/14 14:48,4,call,2014-11,Tesco,mobile 4,15/10/14 17:27,4,call,2014-11,Tesco,mobile 5,15/10/14 18:55,4,call,2014-11,Tesco,mobile 6,16/10/14 06:58,34.429,data,2014-11,data,data 7,16/10/14 15:01,602,call,2014-11,Three,mobile 8,16/10/14 15:12,1050,call,2014-11,Three,mobile 9,16/10/14 15:30,19,call,2014-11,voicemail,voicemail 10,16/10/14 16:21,1183,call,2014-11,Three,mobile 11,16/10/14 22:18,1,sms,2014-11,Meteor,mobile 12,16/10/14 22:21,1,sms,2014-11,Meteor,mobile 13,17/10/14 06:58,34.429,data,2014-11,data,data 14,17/10/14 10:53,1,sms,2014-11,Tesco,mobile 15,17/10/14 11:19,1,sms,2014-11,Tesco,mobile 16,17/10/14 11:20,1,sms,2014-11,Meteor,mobile 17,17/10/14 17:22,1,sms,2014-11,Vodafone,mobile 18,17/10/14 17:23,1,sms,2014-11,Vodafone,mobile 19,17/10/14 17:26,92,call,2014-11,Three,mobile 20,17/10/14 17:29,4,call,2014-11,Vodafone,mobile 21,17/10/14 17:30,375,call,2014-11,Tesco,mobile 22,17/10/14 17:42,1,sms,2014-11,Vodafone,mobile 23,17/10/14 17:44,1,sms,2014-11,Vodafone,mobile 24,17/10/14 17:44,1,sms,2014-11,Vodafone,mobile 25,17/10/14 17:44,1,sms,2014-11,Vodafone,mobile 26,18/10/14 06:58,34.429,data,2014-11,data,data 27,18/10/14 11:51,783,call,2014-11,Tesco,mobile 28,18/10/14 12:06,4,call,2014-11,Vodafone,mobile 29,18/10/14 12:06,3,call,2014-11,Vodafone,mobile 30,18/10/14 13:08,101,call,2014-11,Vodafone,mobile 31,18/10/14 13:10,1714,call,2014-11,Three,mobile 32,18/10/14 14:01,96,call,2014-11,voicemail,voicemail 33,18/10/14 18:52,1,sms,2014-11,Vodafone,mobile 34,18/10/14 20:44,384,call,2014-11,Three,mobile 35,18/10/14 21:04,4,call,2014-11,Three,mobile 36,18/10/14 21:06,1,sms,2014-11,Three,mobile 37,18/10/14 21:23,1,sms,2014-11,Vodafone,mobile 38,18/10/14 22:37,1,sms,2014-11,Three,mobile 39,19/10/14 06:58,34.429,data,2014-11,data,data 40,19/10/14 14:47,53,call,2014-11,Three,mobile 41,19/10/14 15:46,86,call,2014-11,Tesco,mobile 42,19/10/14 16:21,23,call,2014-11,Three,mobile 43,19/10/14 16:30,38,call,2014-11,Three,mobile 44,19/10/14 20:25,428,call,2014-11,Three,mobile 45,20/10/14 06:58,34.429,data,2014-11,data,data 46,20/10/14 09:43,69,call,2014-11,voicemail,voicemail 47,20/10/14 09:43,18,call,2014-11,voicemail,voicemail 48,20/10/14 13:55,3,call,2014-11,Vodafone,mobile 49,20/10/14 13:56,6,call,2014-11,landline,landline 50,20/10/14 18:14,5,call,2014-11,Tesco,mobile 51,20/10/14 18:24,131,call,2014-11,Vodafone,mobile 52,20/10/14 19:59,1,sms,2014-11,Vodafone,mobile 53,20/10/14 20:16,1,sms,2014-11,Vodafone,mobile 54,21/10/14 06:58,34.429,data,2014-11,data,data 55,21/10/14 16:17,550,call,2014-11,Three,mobile 56,22/10/14 06:58,34.429,data,2014-11,data,data 57,22/10/14 12:04,7,call,2014-11,Three,mobile 58,23/10/14 06:58,34.429,data,2014-11,data,data 59,23/10/14 08:34,1940,call,2014-11,landline,landline 60,23/10/14 09:45,281,call,2014-11,Meteor,mobile 61,23/10/14 10:46,1,sms,2014-11,Tesco,mobile 62,23/10/14 10:54,1,sms,2014-11,Vodafone,mobile 63,23/10/14 11:17,1,sms,2014-11,Vodafone,mobile 64,23/10/14 11:25,24,call,2014-11,Vodafone,mobile 65,23/10/14 17:48,263,call,2014-11,Meteor,mobile 66,24/10/14 06:58,34.429,data,2014-11,data,data 67,24/10/14 13:35,1,sms,2014-11,Vodafone,mobile 68,24/10/14 13:39,1,sms,2014-11,Vodafone,mobile 69,24/10/14 13:39,1,sms,2014-11,Vodafone,mobile 70,24/10/14 13:47,1,sms,2014-11,Vodafone,mobile 71,24/10/14 13:48,1,sms,2014-11,Vodafone,mobile 72,24/10/14 13:50,1,sms,2014-11,Vodafone,mobile 73,24/10/14 13:57,1,sms,2014-11,Vodafone,mobile 74,24/10/14 13:57,1,sms,2014-11,Vodafone,mobile 75,24/10/14 14:20,1,sms,2014-11,Three,mobile 76,24/10/14 14:27,1,sms,2014-11,Vodafone,mobile 77,24/10/14 18:29,1,sms,2014-11,Vodafone,mobile 78,24/10/14 18:33,387,call,2014-11,Tesco,mobile 79,24/10/14 18:40,1,sms,2014-11,Vodafone,mobile 80,25/10/14 06:58,34.429,data,2014-11,data,data 81,26/10/14 06:58,34.429,data,2014-11,data,data 82,26/10/14 14:51,4,call,2014-11,Three,mobile 83,26/10/14 21:10,637,call,2014-11,Tesco,mobile 84,26/10/14 21:22,28,call,2014-11,Meteor,mobile 85,26/10/14 21:38,62,call,2014-11,Three,mobile 86,27/10/14 01:45,3,call,2014-11,Tesco,mobile 87,27/10/14 06:58,34.429,data,2014-11,data,data 88,27/10/14 11:03,146,call,2014-11,Three,mobile 89,27/10/14 16:30,48,call,2014-11,Three,mobile 90,27/10/14 19:20,862,call,2014-11,Three,mobile 91,27/10/14 19:55,25,call,2014-11,Three,mobile 92,28/10/14 06:58,34.429,data,2014-11,data,data 93,28/10/14 16:39,833,call,2014-11,landline,landline 94,28/10/14 20:44,206,call,2014-11,Tesco,mobile 95,29/10/14 06:58,34.429,data,2014-11,data,data 96,29/10/14 12:56,442,call,2014-11,Vodafone,mobile 97,30/10/14 06:58,34.429,data,2014-11,data,data 98,30/10/14 14:31,463,call,2014-11,Vodafone,mobile 99,30/10/14 19:48,4,call,2014-11,Vodafone,mobile 100,30/10/14 20:02,4,call,2014-11,Meteor,mobile 101,31/10/14 06:58,34.429,data,2014-11,data,data 102,31/10/14 07:46,1,sms,2014-11,Vodafone,mobile 103,31/10/14 08:00,1,sms,2014-11,Vodafone,mobile 104,31/10/14 13:26,5,call,2014-11,Three,mobile 105,31/10/14 13:27,1234,call,2014-11,Tesco,mobile 106,31/10/14 14:10,43,call,2014-11,Three,mobile 107,31/10/14 18:29,1,sms,2014-11,Three,mobile 108,31/10/14 18:29,1,sms,2014-11,Three,mobile 109,31/10/14 18:30,483,call,2014-11,Meteor,mobile 110,31/10/14 18:39,3,call,2014-11,Tesco,mobile 111,01/11/14 06:58,34.429,data,2014-11,data,data 112,01/11/14 15:13,955,call,2014-11,Vodafone,mobile 113,01/11/14 17:54,4,call,2014-11,Tesco,mobile 114,02/11/14 06:58,34.429,data,2014-11,data,data 115,02/11/14 14:34,459,call,2014-11,Three,mobile 116,02/11/14 15:44,1023,call,2014-11,Three,mobile 117,02/11/14 19:16,1025,call,2014-11,Three,mobile 118,02/11/14 21:42,169,call,2014-11,Meteor,mobile 119,02/11/14 22:55,8,call,2014-11,Meteor,mobile 120,03/11/14 06:58,34.429,data,2014-11,data,data 121,03/11/14 08:40,1,sms,2014-11,special,special 122,03/11/14 10:30,135,call,2014-11,Three,mobile 123,03/11/14 10:37,7,call,2014-11,Vodafone,mobile 124,03/11/14 10:47,37,call,2014-11,Vodafone,mobile 125,03/11/14 14:04,1,sms,2014-11,Vodafone,mobile 126,03/11/14 15:27,5,call,2014-11,Three,mobile 127,03/11/14 16:07,123,call,2014-11,landline,landline 128,03/11/14 16:10,4,call,2014-11,Tesco,mobile 129,03/11/14 17:03,90,call,2014-11,Vodafone,mobile 130,03/11/14 22:36,3,call,2014-11,Tesco,mobile 131,04/11/14 06:58,34.429,data,2014-11,data,data 132,04/11/14 11:58,1,sms,2014-11,Vodafone,mobile 133,04/11/14 11:58,1,sms,2014-11,Vodafone,mobile 134,04/11/14 13:12,1,sms,2014-11,Three,mobile 135,04/11/14 14:05,1,sms,2014-11,Three,mobile 136,04/11/14 14:26,1,call,2014-11,voicemail,voicemail 137,04/11/14 14:26,98,call,2014-11,voicemail,voicemail 138,04/11/14 16:13,1,sms,2014-11,Three,mobile 139,04/11/14 16:19,8,call,2014-11,Meteor,mobile 140,04/11/14 16:22,9,call,2014-11,Meteor,mobile 141,04/11/14 16:24,4,call,2014-11,Meteor,mobile 142,04/11/14 16:34,1,sms,2014-11,Three,mobile 143,04/11/14 16:39,1,sms,2014-11,Meteor,mobile 144,04/11/14 16:41,1,sms,2014-11,Meteor,mobile 145,04/11/14 16:45,1,sms,2014-11,Three,mobile 146,04/11/14 18:26,4,call,2014-11,Tesco,mobile 147,04/11/14 20:15,11,call,2014-11,Three,mobile 148,04/11/14 20:15,1,sms,2014-11,Meteor,mobile 149,04/11/14 20:15,1,sms,2014-11,Meteor,mobile 150,04/11/14 20:16,166,call,2014-11,Meteor,mobile 151,05/11/14 06:58,34.429,data,2014-11,data,data 152,05/11/14 10:59,55,call,2014-11,Three,mobile 153,05/11/14 11:30,1,sms,2014-11,Vodafone,mobile 154,05/11/14 11:43,1,sms,2014-11,Vodafone,mobile 155,05/11/14 12:43,1,sms,2014-11,Vodafone,mobile 156,05/11/14 19:35,29,call,2014-11,Vodafone,mobile 157,06/11/14 01:02,1,sms,2014-11,Vodafone,mobile 158,06/11/14 01:02,1,sms,2014-11,Vodafone,mobile 159,06/11/14 06:58,34.429,data,2014-11,data,data 160,06/11/14 09:04,1,sms,2014-11,Vodafone,mobile 161,06/11/14 09:05,1,sms,2014-11,Vodafone,mobile 162,06/11/14 09:47,150,call,2014-11,Vodafone,mobile 163,06/11/14 09:50,89,call,2014-11,Three,mobile 164,06/11/14 09:52,75,call,2014-11,Meteor,mobile 165,06/11/14 09:54,5,call,2014-11,Three,mobile 166,06/11/14 11:47,8,call,2014-11,Vodafone,mobile 167,06/11/14 14:52,1,sms,2014-11,Vodafone,mobile 168,06/11/14 18:02,279,call,2014-11,Tesco,mobile 169,06/11/14 18:07,452,call,2014-11,Vodafone,mobile 170,07/11/14 06:58,34.429,data,2014-11,data,data 171,07/11/14 09:33,1205,call,2014-11,Vodafone,mobile 172,07/11/14 17:51,218,call,2014-11,Three,mobile 173,07/11/14 21:04,1,sms,2014-11,Vodafone,mobile 174,07/11/14 21:10,1,sms,2014-11,Vodafone,mobile 175,07/11/14 21:12,1,sms,2014-11,Vodafone,mobile 176,07/11/14 21:12,1,sms,2014-11,Vodafone,mobile 177,07/11/14 21:19,1,sms,2014-11,Vodafone,mobile 178,07/11/14 21:25,1,sms,2014-11,Vodafone,mobile 179,07/11/14 21:31,1,sms,2014-11,Vodafone,mobile 180,07/11/14 21:31,1,sms,2014-11,Vodafone,mobile 181,07/11/14 22:04,1,sms,2014-11,Vodafone,mobile 182,08/11/14 06:58,34.429,data,2014-11,data,data 183,08/11/14 16:33,41,call,2014-11,Three,mobile 184,08/11/14 18:18,6,call,2014-11,Tesco,mobile 185,09/11/14 01:41,1,sms,2014-11,Three,mobile 186,09/11/14 01:49,1,sms,2014-11,Three,mobile 187,09/11/14 01:50,1,sms,2014-11,Three,mobile 188,09/11/14 02:04,1,sms,2014-11,Three,mobile 189,09/11/14 06:58,34.429,data,2014-11,data,data 190,09/11/14 19:21,14,call,2014-11,Tesco,mobile 191,09/11/14 22:09,3,call,2014-11,Three,mobile 192,10/11/14 06:58,34.429,data,2014-11,data,data 193,10/11/14 09:29,178,call,2014-11,Vodafone,mobile 194,10/11/14 11:36,412,call,2014-11,Three,mobile 195,10/11/14 14:59,13,call,2014-11,Three,mobile 196,10/11/14 15:15,459,call,2014-11,Three,mobile 197,10/11/14 18:19,1,sms,2014-11,Three,mobile 198,10/11/14 18:34,1,sms,2014-11,Three,mobile 199,11/11/14 06:58,34.429,data,2014-11,data,data 200,11/11/14 09:28,3,call,2014-11,Vodafone,mobile 201,11/11/14 11:32,3,call,2014-11,Vodafone,mobile 202,11/11/14 11:37,1,sms,2014-11,Three,mobile 203,11/11/14 12:39,36,call,2014-11,Three,mobile 204,11/11/14 14:13,1,sms,2014-11,Meteor,mobile 205,11/11/14 14:20,1,sms,2014-11,Meteor,mobile 206,11/11/14 14:41,1,sms,2014-11,Meteor,mobile 207,11/11/14 19:56,1,sms,2014-11,Three,mobile 208,12/11/14 06:58,34.429,data,2014-11,data,data 209,12/11/14 10:48,1,sms,2014-11,Vodafone,mobile 210,12/11/14 10:48,1,sms,2014-11,Three,mobile 211,12/11/14 10:48,1,sms,2014-11,Vodafone,mobile 212,12/11/14 10:49,1,sms,2014-11,Three,mobile 213,12/11/14 12:04,1,sms,2014-11,Three,mobile 214,12/11/14 13:49,1,sms,2014-11,Vodafone,mobile 215,12/11/14 16:16,1,sms,2014-11,Vodafone,mobile 216,12/11/14 16:19,4,call,2014-11,landline,landline 217,12/11/14 16:42,1,sms,2014-11,Vodafone,mobile 218,12/11/14 16:42,1,sms,2014-11,Vodafone,mobile 219,12/11/14 17:14,1,sms,2014-11,Vodafone,mobile 220,12/11/14 17:14,1,sms,2014-11,Vodafone,mobile 221,12/11/14 17:49,1,sms,2014-11,Three,mobile 222,12/11/14 17:56,145,call,2014-11,Three,mobile 223,12/11/14 17:59,1001,call,2014-11,Three,mobile 224,12/11/14 19:01,7,call,2014-11,Vodafone,mobile 225,12/11/14 19:18,1,sms,2014-11,Three,mobile 226,12/11/14 19:18,1,sms,2014-11,Three,mobile 227,12/11/14 19:20,1,sms,2014-11,Vodafone,mobile 228,13/11/14 06:58,34.429,data,2014-12,data,data 229,13/11/14 22:30,1,sms,2014-11,Three,mobile 230,13/11/14 22:31,1,sms,2014-11,Vodafone,mobile 231,14/11/14 06:58,34.429,data,2014-12,data,data 232,14/11/14 17:24,124,call,2014-12,voicemail,voicemail 233,14/11/14 17:28,1,sms,2014-12,Vodafone,mobile 234,15/11/14 06:58,34.429,data,2014-12,data,data 235,16/11/14 06:58,34.429,data,2014-12,data,data 236,16/11/14 14:05,4,call,2014-12,Vodafone,mobile 237,17/11/14 06:58,34.429,data,2014-12,data,data 238,18/11/14 06:58,34.429,data,2014-12,data,data 239,18/11/14 08:22,1,sms,2014-12,Vodafone,mobile 240,18/11/14 08:29,1,sms,2014-12,Vodafone,mobile 241,18/11/14 08:29,1,sms,2014-12,Vodafone,mobile 242,18/11/14 08:33,1,sms,2014-12,Vodafone,mobile 243,18/11/14 08:34,1,sms,2014-12,Vodafone,mobile 244,18/11/14 08:34,1,sms,2014-12,Vodafone,mobile 245,18/11/14 08:39,1,sms,2014-12,Vodafone,mobile 246,18/11/14 08:42,1,sms,2014-12,Vodafone,mobile 247,18/11/14 08:45,1,sms,2014-12,Vodafone,mobile 248,18/11/14 09:35,1,sms,2014-12,Vodafone,mobile 249,19/11/14 06:58,34.429,data,2014-12,data,data 250,19/11/14 14:05,128,call,2014-12,Tesco,mobile 251,19/11/14 14:11,249,call,2014-12,Meteor,mobile 252,19/11/14 18:56,2120,call,2014-12,Three,mobile 253,19/11/14 22:48,1,sms,2014-12,Vodafone,mobile 254,20/11/14 06:58,34.429,data,2014-12,data,data 255,20/11/14 14:57,71,call,2014-12,voicemail,voicemail 256,20/11/14 14:59,56,call,2014-12,Three,mobile 257,20/11/14 16:22,1,sms,2014-12,Meteor,mobile 258,20/11/14 19:08,9,call,2014-12,Three,mobile 259,20/11/14 21:03,3,call,2014-12,Three,mobile 260,20/11/14 21:03,3,call,2014-12,Three,mobile 261,21/11/14 00:17,17,call,2014-12,Tesco,mobile 262,21/11/14 01:13,1,sms,2014-12,Vodafone,mobile 263,21/11/14 06:58,34.429,data,2014-12,data,data 264,21/11/14 10:29,1,sms,2014-12,Meteor,mobile 265,21/11/14 10:29,1,sms,2014-12,Meteor,mobile 266,21/11/14 10:30,1,sms,2014-12,Meteor,mobile 267,21/11/14 11:29,8,call,2014-12,landline,landline 268,21/11/14 11:31,982,call,2014-12,Vodafone,mobile 269,21/11/14 11:49,11,call,2014-12,landline,landline 270,21/11/14 11:50,34,call,2014-12,Three,mobile 271,21/11/14 11:50,8,call,2014-12,Three,mobile 272,21/11/14 13:31,600,call,2014-12,Tesco,mobile 273,21/11/14 18:07,244,call,2014-12,Meteor,mobile 274,22/11/14 02:10,186,call,2014-12,Tesco,mobile 275,22/11/14 06:58,34.429,data,2014-12,data,data 276,22/11/14 12:02,75,call,2014-12,Meteor,mobile 277,22/11/14 12:10,90,call,2014-12,Vodafone,mobile 278,22/11/14 14:30,13,call,2014-12,Vodafone,mobile 279,22/11/14 14:33,20,call,2014-12,Vodafone,mobile 280,22/11/14 14:34,2,call,2014-12,Vodafone,mobile 281,23/11/14 06:58,34.429,data,2014-12,data,data 282,23/11/14 13:24,208,call,2014-12,Three,mobile 283,23/11/14 16:10,107,call,2014-12,Three,mobile 284,23/11/14 17:36,55,call,2014-12,Three,mobile 285,23/11/14 17:53,5,call,2014-12,Three,mobile 286,23/11/14 17:53,20,call,2014-12,Three,mobile 287,23/11/14 17:54,2,call,2014-12,Three,mobile 288,24/11/14 06:58,34.429,data,2014-12,data,data 289,24/11/14 09:40,1,sms,2014-12,Three,mobile 290,24/11/14 12:24,4,call,2014-12,Meteor,mobile 291,25/11/14 06:58,34.429,data,2014-12,data,data 292,25/11/14 11:25,21,call,2014-12,Vodafone,mobile 293,25/11/14 16:09,1,sms,2014-12,Meteor,mobile 294,25/11/14 16:19,1,sms,2014-12,Meteor,mobile 295,25/11/14 17:10,114,call,2014-12,Meteor,mobile 296,25/11/14 18:06,1,sms,2014-12,Meteor,mobile 297,25/11/14 18:18,81,call,2014-12,Meteor,mobile 298,25/11/14 18:47,174,call,2014-12,voicemail,voicemail 299,25/11/14 19:10,71,call,2014-12,Tesco,mobile 300,25/11/14 19:20,40,call,2014-12,Tesco,mobile 301,25/11/14 19:21,29,call,2014-12,voicemail,voicemail 302,25/11/14 19:25,37,call,2014-12,Tesco,mobile 303,25/11/14 19:26,63,call,2014-12,voicemail,voicemail 304,25/11/14 20:39,1,sms,2014-12,Three,mobile 305,26/11/14 06:58,34.429,data,2014-12,data,data 306,26/11/14 07:03,14,call,2014-12,Meteor,mobile 307,26/11/14 07:15,1,sms,2014-12,Vodafone,mobile 308,26/11/14 07:57,1,sms,2014-12,Vodafone,mobile 309,26/11/14 07:59,4,call,2014-12,Three,mobile 310,26/11/14 08:00,1,sms,2014-12,Vodafone,mobile 311,26/11/14 08:13,10,call,2014-12,Three,mobile 312,26/11/14 08:16,3,call,2014-12,Three,mobile 313,26/11/14 08:26,3,call,2014-12,Three,mobile 314,26/11/14 08:27,3,call,2014-12,Three,mobile 315,26/11/14 09:01,1,sms,2014-12,Meteor,mobile 316,26/11/14 11:53,1,sms,2014-12,Meteor,mobile 317,26/11/14 11:54,1,sms,2014-12,Meteor,mobile 318,26/11/14 11:54,1,sms,2014-12,Meteor,mobile 319,26/11/14 11:56,1,sms,2014-12,Meteor,mobile 320,26/11/14 17:48,3,call,2014-12,Three,mobile 321,27/11/14 06:58,34.429,data,2014-12,data,data 322,27/11/14 16:53,1116,call,2014-12,Three,mobile 323,27/11/14 18:38,1,sms,2014-12,Three,mobile 324,28/11/14 06:58,34.429,data,2014-12,data,data 325,28/11/14 13:05,1,sms,2014-12,Three,mobile 326,28/11/14 13:12,1,sms,2014-12,Three,mobile 327,28/11/14 19:03,143,call,2014-12,Tesco,mobile 328,29/11/14 06:58,34.429,data,2014-12,data,data 329,29/11/14 14:44,151,call,2014-12,Three,mobile 330,30/11/14 06:58,34.429,data,2014-12,data,data 331,30/11/14 11:45,1,sms,2014-12,Three,mobile 332,30/11/14 11:48,1,sms,2014-12,Three,mobile 333,30/11/14 11:48,1,sms,2014-12,Three,mobile 334,30/11/14 12:06,1,sms,2014-12,Three,mobile 335,30/11/14 14:24,1,sms,2014-12,Three,mobile 336,30/11/14 14:44,1,sms,2014-12,Three,mobile 337,30/11/14 14:51,4,call,2014-12,Three,mobile 338,01/12/14 06:58,34.429,data,2014-12,data,data 339,01/12/14 12:51,1,sms,2014-12,Three,mobile 340,01/12/14 12:59,1,sms,2014-12,Three,mobile 341,02/12/14 06:58,34.429,data,2014-12,data,data 342,02/12/14 11:40,526,call,2014-12,Meteor,mobile 343,03/12/14 06:58,34.429,data,2014-12,data,data 344,03/12/14 15:01,844,call,2014-12,landline,landline 345,03/12/14 18:10,383,call,2014-12,Tesco,mobile 346,04/12/14 06:58,34.429,data,2014-12,data,data 347,04/12/14 13:52,6,call,2014-12,Vodafone,mobile 348,04/12/14 15:34,37,call,2014-12,Three,mobile 349,04/12/14 16:02,15,call,2014-12,Meteor,mobile 350,04/12/14 23:41,71,call,2014-12,voicemail,voicemail 351,05/12/14 06:58,34.429,data,2014-12,data,data 352,05/12/14 16:49,465,call,2014-12,landline,landline 353,05/12/14 18:17,153,call,2014-12,Tesco,mobile 354,05/12/14 18:25,826,call,2014-12,Three,mobile 355,06/12/14 06:58,34.429,data,2014-12,data,data 356,06/12/14 11:33,442,call,2014-12,Meteor,mobile 357,06/12/14 18:25,1,sms,2014-12,Vodafone,mobile 358,06/12/14 18:26,1,sms,2014-12,Tesco,mobile 359,06/12/14 18:26,1,sms,2014-12,Vodafone,mobile 360,06/12/14 18:27,1,sms,2014-12,world,world 361,06/12/14 18:28,1,sms,2014-12,world,world 362,06/12/14 19:40,191,call,2014-12,Meteor,mobile 363,07/12/14 06:58,34.429,data,2014-12,data,data 364,07/12/14 13:03,99,call,2014-12,voicemail,voicemail 365,07/12/14 13:45,428,call,2014-12,Three,mobile 366,07/12/14 14:39,3,call,2014-12,Three,mobile 367,07/12/14 20:23,727,call,2014-12,Three,mobile 368,07/12/14 20:36,33,call,2014-12,Tesco,mobile 369,07/12/14 20:37,120,call,2014-12,Three,mobile 370,07/12/14 23:22,1,sms,2014-12,world,world 371,07/12/14 23:22,1,sms,2014-12,world,world 372,08/12/14 06:58,34.429,data,2014-12,data,data 373,08/12/14 17:38,55,call,2014-12,Meteor,mobile 374,09/12/14 06:58,34.429,data,2014-12,data,data 375,09/12/14 18:32,28,call,2014-12,Tesco,mobile 376,10/12/14 06:58,34.429,data,2014-12,data,data 377,11/12/14 06:58,34.429,data,2014-12,data,data 378,12/12/14 06:58,34.429,data,2014-12,data,data 379,12/12/14 11:00,112,call,2014-12,Vodafone,mobile 380,12/12/14 18:14,52,call,2014-12,Vodafone,mobile 381,13/12/14 06:58,34.429,data,2015-01,data,data 382,13/12/14 14:56,223,call,2014-12,Three,mobile 383,14/12/14 02:05,14,call,2014-12,landline,landline 384,14/12/14 02:07,8,call,2014-12,landline,landline 385,14/12/14 02:09,74,call,2014-12,landline,landline 386,14/12/14 06:58,34.429,data,2015-01,data,data 387,14/12/14 15:28,59,call,2014-12,voicemail,voicemail 388,14/12/14 19:54,25,call,2014-12,Three,mobile 389,15/12/14 06:58,34.429,data,2015-01,data,data 390,15/12/14 19:56,1,sms,2015-01,Three,mobile 391,15/12/14 19:58,1,sms,2015-01,Three,mobile 392,15/12/14 20:03,4,call,2015-01,Three,mobile 393,15/12/14 20:10,1,sms,2015-01,Vodafone,mobile 394,15/12/14 20:10,1,sms,2015-01,Three,mobile 395,15/12/14 23:12,1,sms,2015-01,Three,mobile 396,16/12/14 06:58,34.429,data,2015-01,data,data 397,17/12/14 06:58,34.429,data,2015-01,data,data 398,17/12/14 18:08,1859,call,2015-01,Vodafone,mobile 399,17/12/14 23:26,1,sms,2015-01,Vodafone,mobile 400,18/12/14 06:58,34.429,data,2015-01,data,data 401,18/12/14 12:36,61,call,2015-01,Three,mobile 402,18/12/14 15:46,268,call,2015-01,Meteor,mobile 403,18/12/14 15:57,192,call,2015-01,Meteor,mobile 404,18/12/14 16:10,14,call,2015-01,Meteor,mobile 405,18/12/14 17:54,17,call,2015-01,Three,mobile 406,18/12/14 19:05,46,call,2015-01,Tesco,mobile 407,18/12/14 21:58,4,call,2015-01,Meteor,mobile 408,18/12/14 21:59,4,call,2015-01,Meteor,mobile 409,19/12/14 06:58,34.429,data,2015-01,data,data 410,19/12/14 08:57,1,sms,2015-01,Vodafone,mobile 411,19/12/14 10:14,41,call,2015-01,Three,mobile 412,19/12/14 12:40,3,call,2015-01,Three,mobile 413,19/12/14 12:41,217,call,2015-01,Tesco,mobile 414,19/12/14 12:41,18,call,2015-01,Three,mobile 415,19/12/14 14:48,58,call,2015-01,Meteor,mobile 416,19/12/14 16:49,48,call,2015-01,Tesco,mobile 417,19/12/14 16:51,543,call,2015-01,Tesco,mobile 418,19/12/14 17:00,131,call,2015-01,Three,mobile 419,19/12/14 18:44,1,sms,2015-01,Vodafone,mobile 420,20/12/14 06:58,34.429,data,2015-01,data,data 421,20/12/14 14:39,1,sms,2015-01,Tesco,mobile 422,20/12/14 15:20,1,sms,2015-01,Tesco,mobile 423,20/12/14 15:53,553,call,2015-01,Meteor,mobile 424,20/12/14 16:09,1,sms,2015-01,Tesco,mobile 425,21/12/14 00:05,54,call,2015-01,Meteor,mobile 426,21/12/14 06:58,34.429,data,2015-01,data,data 427,22/12/14 06:58,34.429,data,2015-01,data,data 428,22/12/14 10:42,489,call,2015-01,Tesco,mobile 429,22/12/14 11:22,1,sms,2015-01,Vodafone,mobile 430,22/12/14 11:22,1,sms,2015-01,Meteor,mobile 431,22/12/14 13:33,46,call,2015-01,Vodafone,mobile 432,22/12/14 14:09,1,sms,2015-01,Vodafone,mobile 433,22/12/14 14:15,47,call,2015-01,Vodafone,mobile 434,22/12/14 18:03,1,sms,2015-01,Vodafone,mobile 435,22/12/14 18:03,1,sms,2015-01,Vodafone,mobile 436,22/12/14 18:03,1,sms,2015-01,Vodafone,mobile 437,22/12/14 19:10,1,sms,2015-01,Vodafone,mobile 438,22/12/14 19:12,566,call,2015-01,Tesco,mobile 439,22/12/14 19:35,1,sms,2015-01,Meteor,mobile 440,22/12/14 19:36,1,sms,2015-01,Meteor,mobile 441,22/12/14 23:12,956,call,2015-01,Three,mobile 442,23/12/14 00:57,55,call,2015-01,Tesco,mobile 443,23/12/14 06:58,34.429,data,2015-01,data,data 444,23/12/14 09:17,145,call,2015-01,voicemail,voicemail 445,23/12/14 11:03,40,call,2015-01,landline,landline 446,23/12/14 12:49,449,call,2015-01,Three,mobile 447,23/12/14 15:39,1,sms,2015-01,Vodafone,mobile 448,23/12/14 15:40,37,call,2015-01,landline,landline 449,23/12/14 15:43,1,sms,2015-01,Vodafone,mobile 450,23/12/14 19:45,39,call,2015-01,voicemail,voicemail 451,23/12/14 20:02,28,call,2015-01,landline,landline 452,23/12/14 21:06,1,sms,2015-01,Vodafone,mobile 453,24/12/14 06:58,34.429,data,2015-01,data,data 454,24/12/14 13:22,4,call,2015-01,Three,mobile 455,24/12/14 13:29,234,call,2015-01,Tesco,mobile 456,24/12/14 13:56,165,call,2015-01,Three,mobile 457,24/12/14 13:56,3,call,2015-01,Three,mobile 458,24/12/14 17:06,1,sms,2015-01,Vodafone,mobile 459,24/12/14 17:07,37,call,2015-01,Vodafone,mobile 460,24/12/14 18:44,5,call,2015-01,Meteor,mobile 461,24/12/14 20:44,4,call,2015-01,Meteor,mobile 462,24/12/14 23:34,1,sms,2015-01,Three,mobile 463,25/12/14 06:58,34.429,data,2015-01,data,data 464,25/12/14 12:27,1,sms,2015-01,Vodafone,mobile 465,26/12/14 06:58,34.429,data,2015-01,data,data 466,26/12/14 11:09,101,call,2015-01,voicemail,voicemail 467,26/12/14 11:48,1,sms,2015-01,Vodafone,mobile 468,27/12/14 06:58,34.429,data,2015-01,data,data 469,27/12/14 22:30,1,sms,2015-01,Meteor,mobile 470,27/12/14 22:30,1,sms,2015-01,Meteor,mobile 471,27/12/14 22:30,1,sms,2015-01,Meteor,mobile 472,27/12/14 22:30,1,sms,2015-01,Meteor,mobile 473,28/12/14 06:58,34.429,data,2015-01,data,data 474,29/12/14 06:58,34.429,data,2015-01,data,data 475,29/12/14 12:09,368,call,2015-01,Tesco,mobile 476,30/12/14 06:58,34.429,data,2015-01,data,data 477,30/12/14 11:57,1,sms,2015-01,Three,mobile 478,30/12/14 11:57,1,sms,2015-01,Three,mobile 479,30/12/14 12:01,1,sms,2015-01,Three,mobile 480,30/12/14 12:01,1,sms,2015-01,Three,mobile 481,30/12/14 12:02,1,sms,2015-01,Three,mobile 482,30/12/14 12:03,1,sms,2015-01,Three,mobile 483,30/12/14 12:04,1,sms,2015-01,Three,mobile 484,30/12/14 12:04,1,sms,2015-01,Three,mobile 485,30/12/14 12:05,1,sms,2015-01,Three,mobile 486,30/12/14 12:05,1,sms,2015-01,Three,mobile 487,30/12/14 12:05,1,sms,2015-01,Three,mobile 488,30/12/14 12:05,1,sms,2015-01,Three,mobile 489,30/12/14 12:06,1,sms,2015-01,Three,mobile 490,30/12/14 12:10,1,sms,2015-01,Three,mobile 491,30/12/14 12:10,1,sms,2015-01,Three,mobile 492,30/12/14 12:10,1,sms,2015-01,Three,mobile 493,30/12/14 12:13,1,sms,2015-01,Three,mobile 494,30/12/14 12:14,1,sms,2015-01,Three,mobile 495,30/12/14 12:14,1,sms,2015-01,Three,mobile 496,31/12/14 06:58,34.429,data,2015-01,data,data 497,31/12/14 13:00,5,call,2015-01,Meteor,mobile 498,31/12/14 13:03,358,call,2015-01,Meteor,mobile 499,31/12/14 13:49,526,call,2015-01,landline,landline 500,31/12/14 23:05,1,sms,2015-01,Vodafone,mobile 501,31/12/14 23:37,1,sms,2015-01,Vodafone,mobile 502,31/12/14 23:37,1,sms,2015-01,Vodafone,mobile 503,31/12/14 23:37,1,sms,2015-01,Vodafone,mobile 504,01/01/15 06:58,34.429,data,2015-01,data,data 505,02/01/15 06:58,34.429,data,2015-01,data,data 506,02/01/15 11:27,640,call,2015-01,Vodafone,mobile 507,02/01/15 23:26,1,sms,2015-01,Meteor,mobile 508,02/01/15 23:28,1,sms,2015-01,Meteor,mobile 509,03/01/15 06:58,34.429,data,2015-01,data,data 510,03/01/15 12:01,158,call,2015-01,Vodafone,mobile 511,04/01/15 00:57,104,call,2015-01,Three,mobile 512,04/01/15 06:58,34.429,data,2015-01,data,data 513,04/01/15 14:20,3,call,2015-01,Vodafone,mobile 514,04/01/15 14:31,6,call,2015-01,Vodafone,mobile 515,04/01/15 14:32,41,call,2015-01,Vodafone,mobile 516,05/01/15 06:58,34.429,data,2015-01,data,data 517,05/01/15 09:49,56,call,2015-01,landline,landline 518,05/01/15 09:51,128,call,2015-01,landline,landline 519,05/01/15 10:10,77,call,2015-01,Three,mobile 520,05/01/15 10:25,5,call,2015-01,Three,mobile 521,05/01/15 10:25,1,sms,2015-01,Three,mobile 522,05/01/15 10:52,1,sms,2015-01,Three,mobile 523,05/01/15 10:56,144,call,2015-01,landline,landline 524,05/01/15 11:58,99,call,2015-01,Meteor,mobile 525,05/01/15 14:29,60,call,2015-01,landline,landline 526,05/01/15 16:41,682,call,2015-01,Three,mobile 527,05/01/15 17:22,36,call,2015-01,Tesco,mobile 528,05/01/15 20:23,734,call,2015-01,Three,mobile 529,06/01/15 06:58,34.429,data,2015-01,data,data 530,06/01/15 09:04,1,sms,2015-01,Three,mobile 531,06/01/15 09:04,1,sms,2015-01,Three,mobile 532,06/01/15 13:28,16,call,2015-01,Meteor,mobile 533,06/01/15 13:29,295,call,2015-01,Vodafone,mobile 534,06/01/15 13:55,1,sms,2015-01,Vodafone,mobile 535,06/01/15 19:17,106,call,2015-01,Meteor,mobile 536,06/01/15 20:40,29,call,2015-01,Vodafone,mobile 537,07/01/15 06:58,34.429,data,2015-01,data,data 538,07/01/15 09:28,1,sms,2015-01,Vodafone,mobile 539,07/01/15 21:20,1,sms,2015-01,Vodafone,mobile 540,07/01/15 21:20,1,sms,2015-01,Vodafone,mobile 541,08/01/15 06:58,34.429,data,2015-01,data,data 542,08/01/15 15:02,4,call,2015-01,Meteor,mobile 543,08/01/15 15:10,3,call,2015-01,Meteor,mobile 544,08/01/15 15:10,23,call,2015-01,Meteor,mobile 545,08/01/15 20:15,290,call,2015-01,Tesco,mobile 546,08/01/15 20:26,1,sms,2015-01,Vodafone,mobile 547,08/01/15 20:30,12,call,2015-01,Tesco,mobile 548,08/01/15 20:31,1247,call,2015-01,Three,mobile 549,08/01/15 22:41,1,sms,2015-01,Vodafone,mobile 550,08/01/15 22:41,1,sms,2015-01,Vodafone,mobile 551,08/01/15 22:52,1,sms,2015-01,Vodafone,mobile 552,08/01/15 22:52,1,sms,2015-01,Vodafone,mobile 553,08/01/15 23:06,1,sms,2015-01,Vodafone,mobile 554,08/01/15 23:06,1,sms,2015-01,Vodafone,mobile 555,09/01/15 06:58,34.429,data,2015-01,data,data 556,09/01/15 09:25,1,sms,2015-01,Meteor,mobile 557,09/01/15 09:43,33,call,2015-01,Three,mobile 558,09/01/15 10:07,4,call,2015-01,Vodafone,mobile 559,09/01/15 17:32,57,call,2015-01,Vodafone,mobile 560,10/01/15 06:58,34.429,data,2015-01,data,data 561,10/01/15 14:10,2,call,2015-01,Three,mobile 562,10/01/15 14:36,6,call,2015-01,Vodafone,mobile 563,10/01/15 14:44,398,call,2015-01,Vodafone,mobile 564,10/01/15 15:58,412,call,2015-01,Meteor,mobile 565,10/01/15 16:57,568,call,2015-01,Three,mobile 566,10/01/15 21:16,1,sms,2015-01,Vodafone,mobile 567,10/01/15 21:16,1,sms,2015-01,Vodafone,mobile 568,11/01/15 06:58,34.429,data,2015-01,data,data 569,11/01/15 13:29,1,sms,2015-01,Vodafone,mobile 570,11/01/15 13:54,201,call,2015-01,Three,mobile 571,12/01/15 06:58,34.429,data,2015-01,data,data 572,12/01/15 12:01,18,call,2015-01,Meteor,mobile 573,12/01/15 12:01,7,call,2015-01,Meteor,mobile 574,12/01/15 18:23,4,call,2015-01,Three,mobile 575,12/01/15 18:26,1,sms,2015-01,Vodafone,mobile 576,12/01/15 18:26,1,sms,2015-01,Vodafone,mobile 577,13/01/15 06:58,34.429,data,2015-02,data,data 578,13/01/15 15:04,503,call,2015-01,Three,mobile 579,13/01/15 19:09,1,sms,2015-01,Three,mobile 580,13/01/15 19:44,1,sms,2015-01,Three,mobile 581,13/01/15 19:44,1,sms,2015-01,Three,mobile 582,13/01/15 19:57,1,sms,2015-01,Vodafone,mobile 583,13/01/15 19:57,1,sms,2015-01,Vodafone,mobile 584,13/01/15 19:58,105,call,2015-01,landline,landline 585,13/01/15 20:00,466,call,2015-01,landline,landline 586,14/01/15 06:58,34.429,data,2015-02,data,data 587,14/01/15 17:15,13,call,2015-01,landline,landline 588,14/01/15 19:16,397,call,2015-01,Three,mobile 589,14/01/15 20:47,36,call,2015-01,Three,mobile 590,14/01/15 23:34,1,sms,2015-01,Vodafone,mobile 591,14/01/15 23:34,1,sms,2015-01,Vodafone,mobile 592,14/01/15 23:35,1,sms,2015-01,Three,mobile 593,14/01/15 23:36,1,sms,2015-01,Three,mobile 594,15/01/15 06:58,34.429,data,2015-02,data,data 595,15/01/15 10:36,28,call,2015-02,Three,mobile 596,15/01/15 12:23,1,sms,2015-02,special,special 597,15/01/15 17:22,168,call,2015-02,Tesco,mobile 598,16/01/15 06:58,34.429,data,2015-02,data,data 599,16/01/15 09:45,1,call,2015-02,Meteor,mobile 600,16/01/15 09:56,61,call,2015-02,Meteor,mobile 601,16/01/15 10:17,14,call,2015-02,Three,mobile 602,16/01/15 10:25,20,call,2015-02,Three,mobile 603,16/01/15 17:46,411,call,2015-02,Tesco,mobile 604,16/01/15 18:07,1,sms,2015-02,Vodafone,mobile 605,16/01/15 18:07,1,sms,2015-02,Vodafone,mobile 606,16/01/15 18:07,1,sms,2015-02,Vodafone,mobile 607,16/01/15 18:07,1,sms,2015-02,Vodafone,mobile 608,16/01/15 18:07,1,sms,2015-02,Three,mobile 609,16/01/15 18:07,1,sms,2015-02,Three,mobile 610,17/01/15 06:58,34.429,data,2015-02,data,data 611,17/01/15 18:50,78,call,2015-02,Three,mobile 612,17/01/15 21:59,82,call,2015-02,Tesco,mobile 613,18/01/15 06:58,34.429,data,2015-02,data,data 614,18/01/15 16:27,478,call,2015-02,Three,mobile 615,18/01/15 17:04,700,call,2015-02,Tesco,mobile 616,19/01/15 06:58,34.429,data,2015-02,data,data 617,19/01/15 12:44,1,sms,2015-02,Vodafone,mobile 618,19/01/15 19:57,103,call,2015-02,Tesco,mobile 619,19/01/15 20:08,53,call,2015-02,Tesco,mobile 620,19/01/15 20:14,38,call,2015-02,Tesco,mobile 621,20/01/15 06:58,34.429,data,2015-02,data,data 622,20/01/15 15:08,1,sms,2015-02,Vodafone,mobile 623,20/01/15 19:49,1,sms,2015-02,Vodafone,mobile 624,20/01/15 20:23,1,sms,2015-02,Vodafone,mobile 625,21/01/15 06:58,34.429,data,2015-02,data,data 626,21/01/15 10:13,48,call,2015-02,Three,mobile 627,21/01/15 14:36,93,call,2015-02,voicemail,voicemail 628,21/01/15 14:44,1,sms,2015-02,Vodafone,mobile 629,21/01/15 15:56,27,call,2015-02,voicemail,voicemail 630,21/01/15 15:57,265,call,2015-02,Three,mobile 631,21/01/15 18:04,777,call,2015-02,Tesco,mobile 632,21/01/15 19:38,1090,call,2015-02,Meteor,mobile 633,21/01/15 19:59,491,call,2015-02,Vodafone,mobile 634,22/01/15 06:58,34.429,data,2015-02,data,data 635,22/01/15 18:59,41,call,2015-02,voicemail,voicemail 636,22/01/15 19:00,1107,call,2015-02,Vodafone,mobile 637,23/01/15 06:58,34.429,data,2015-02,data,data 638,23/01/15 12:56,29,call,2015-02,landline,landline 639,23/01/15 14:32,200,call,2015-02,Tesco,mobile 640,23/01/15 15:09,129,call,2015-02,Three,mobile 641,23/01/15 15:22,1,sms,2015-02,Vodafone,mobile 642,23/01/15 15:24,1,sms,2015-02,Vodafone,mobile 643,23/01/15 15:37,1,sms,2015-02,Vodafone,mobile 644,23/01/15 21:22,206,call,2015-02,Tesco,mobile 645,24/01/15 06:58,34.429,data,2015-02,data,data 646,25/01/15 06:58,34.429,data,2015-02,data,data 647,25/01/15 09:16,4,call,2015-02,Vodafone,mobile 648,25/01/15 16:55,1863,call,2015-02,Three,mobile 649,26/01/15 06:58,34.429,data,2015-02,data,data 650,26/01/15 16:54,104,call,2015-02,Three,mobile 651,26/01/15 17:15,501,call,2015-02,Three,mobile 652,27/01/15 06:58,34.429,data,2015-02,data,data 653,27/01/15 09:36,37,call,2015-02,Three,mobile 654,27/01/15 10:55,36,call,2015-02,Meteor,mobile 655,28/01/15 06:58,34.429,data,2015-02,data,data 656,28/01/15 09:44,8,call,2015-02,Vodafone,mobile 657,28/01/15 10:02,7,call,2015-02,landline,landline 658,28/01/15 15:53,272,call,2015-02,landline,landline 659,29/01/15 06:58,34.429,data,2015-02,data,data 660,29/01/15 11:35,1,sms,2015-02,Vodafone,mobile 661,29/01/15 11:50,1,sms,2015-02,Vodafone,mobile 662,29/01/15 17:11,255,call,2015-02,Tesco,mobile 663,29/01/15 17:19,1,sms,2015-02,Vodafone,mobile 664,29/01/15 17:58,362,call,2015-02,Three,mobile 665,29/01/15 18:05,100,call,2015-02,Tesco,mobile 666,29/01/15 19:27,74,call,2015-02,Tesco,mobile 667,30/01/15 06:58,34.429,data,2015-02,data,data 668,30/01/15 19:43,33,call,2015-02,Tesco,mobile 669,30/01/15 19:56,45,call,2015-02,Tesco,mobile 670,31/01/15 06:58,34.429,data,2015-02,data,data 671,31/01/15 12:48,31,call,2015-02,Tesco,mobile 672,31/01/15 13:14,7,call,2015-02,Vodafone,mobile 673,01/02/15 06:58,34.429,data,2015-02,data,data 674,01/02/15 13:33,103,call,2015-02,landline,landline 675,02/02/15 06:58,34.429,data,2015-02,data,data 676,02/02/15 17:11,280,call,2015-02,Tesco,mobile 677,02/02/15 17:16,183,call,2015-02,Tesco,mobile 678,02/02/15 17:35,1,sms,2015-02,Tesco,mobile 679,02/02/15 17:35,1,sms,2015-02,Tesco,mobile 680,02/02/15 17:35,1,sms,2015-02,Three,mobile 681,02/02/15 17:35,1,sms,2015-02,Three,mobile 682,02/02/15 18:17,7,call,2015-02,landline,landline 683,03/02/15 06:58,34.429,data,2015-02,data,data 684,03/02/15 14:45,6,call,2015-02,Three,mobile 685,04/02/15 06:58,34.429,data,2015-02,data,data 686,04/02/15 12:36,21,call,2015-02,voicemail,voicemail 687,04/02/15 14:52,227,call,2015-02,Tesco,mobile 688,04/02/15 17:04,1,sms,2015-02,special,special 689,05/02/15 06:58,34.429,data,2015-02,data,data 690,05/02/15 13:37,62,call,2015-02,voicemail,voicemail 691,05/02/15 13:38,91,call,2015-02,landline,landline 692,06/02/15 06:58,34.429,data,2015-02,data,data 693,06/02/15 10:36,24,call,2015-02,voicemail,voicemail 694,06/02/15 10:37,128,call,2015-02,Vodafone,mobile 695,06/02/15 18:39,23,call,2015-02,Three,mobile 696,06/02/15 18:41,51,call,2015-02,Three,mobile 697,07/02/15 06:58,34.429,data,2015-02,data,data 698,07/02/15 09:48,1,sms,2015-02,Vodafone,mobile 699,07/02/15 09:48,1,sms,2015-02,Vodafone,mobile 700,07/02/15 10:03,119,call,2015-02,Vodafone,mobile 701,07/02/15 11:13,1,sms,2015-02,Vodafone,mobile 702,07/02/15 11:37,4,call,2015-02,Three,mobile 703,07/02/15 15:04,1,sms,2015-02,Vodafone,mobile 704,07/02/15 16:06,1,sms,2015-02,Three,mobile 705,07/02/15 16:11,1,sms,2015-02,Three,mobile 706,07/02/15 16:11,1,sms,2015-02,Three,mobile 707,07/02/15 16:16,1,sms,2015-02,Three,mobile 708,07/02/15 16:27,141,call,2015-02,landline,landline 709,07/02/15 17:33,795,call,2015-02,Three,mobile 710,07/02/15 17:56,121,call,2015-02,Tesco,mobile 711,07/02/15 18:23,2,call,2015-02,Three,mobile 712,07/02/15 22:18,1,sms,2015-02,Three,mobile 713,07/02/15 22:30,1,sms,2015-02,Three,mobile 714,07/02/15 22:30,1,sms,2015-02,Three,mobile 715,08/02/15 06:58,34.429,data,2015-02,data,data 716,08/02/15 20:54,80,call,2015-02,landline,landline 717,09/02/15 06:58,34.429,data,2015-02,data,data 718,09/02/15 09:08,653,call,2015-02,Three,mobile 719,09/02/15 17:41,729,call,2015-02,Three,mobile 720,09/02/15 17:54,89,call,2015-02,Three,mobile 721,09/02/15 18:32,1,sms,2015-02,Vodafone,mobile 722,09/02/15 22:54,1,sms,2015-02,Meteor,mobile 723,10/02/15 00:24,1,sms,2015-02,Vodafone,mobile 724,10/02/15 00:24,1,sms,2015-02,Vodafone,mobile 725,10/02/15 06:58,34.429,data,2015-02,data,data 726,10/02/15 21:40,1,sms,2015-02,Vodafone,mobile 727,11/02/15 06:58,34.429,data,2015-02,data,data 728,12/02/15 06:58,34.429,data,2015-02,data,data 729,12/02/15 20:15,69,call,2015-03,landline,landline 730,12/02/15 20:51,86,call,2015-03,Tesco,mobile 731,13/02/15 06:58,34.429,data,2015-03,data,data 732,13/02/15 10:58,451,call,2015-03,Vodafone,mobile 733,13/02/15 21:13,8,call,2015-03,Vodafone,mobile 734,14/02/15 06:58,34.429,data,2015-03,data,data 735,14/02/15 15:40,106,call,2015-03,Three,mobile 736,14/02/15 16:06,148,call,2015-03,Tesco,mobile 737,15/02/15 06:58,34.429,data,2015-03,data,data 738,15/02/15 18:44,21,call,2015-03,landline,landline 739,16/02/15 06:58,34.429,data,2015-03,data,data 740,17/02/15 06:58,34.429,data,2015-03,data,data 741,17/02/15 15:59,24,call,2015-03,Meteor,mobile 742,17/02/15 19:09,2328,call,2015-03,Three,mobile 743,18/02/15 06:58,34.429,data,2015-03,data,data 744,18/02/15 18:49,165,call,2015-03,Tesco,mobile 745,18/02/15 19:56,94,call,2015-03,Three,mobile 746,19/02/15 06:58,34.429,data,2015-03,data,data 747,19/02/15 18:46,1,sms,2015-03,Vodafone,mobile 748,19/02/15 22:00,1,sms,2015-03,Vodafone,mobile 749,19/02/15 22:00,1,sms,2015-03,Vodafone,mobile 750,19/02/15 22:00,1,sms,2015-03,Vodafone,mobile 751,20/02/15 06:58,34.429,data,2015-03,data,data 752,20/02/15 13:45,68,call,2015-03,Vodafone,mobile 753,21/02/15 06:58,34.429,data,2015-03,data,data 754,22/02/15 06:58,34.429,data,2015-03,data,data 755,23/02/15 06:58,34.429,data,2015-03,data,data 756,23/02/15 20:49,182,call,2015-03,landline,landline 757,24/02/15 06:58,34.429,data,2015-03,data,data 758,24/02/15 10:05,107,call,2015-03,voicemail,voicemail 759,24/02/15 13:32,1,sms,2015-03,Three,mobile 760,24/02/15 13:32,1,sms,2015-03,Three,mobile 761,25/02/15 06:58,34.429,data,2015-03,data,data 762,25/02/15 12:56,117,call,2015-03,Tesco,mobile 763,25/02/15 12:58,129,call,2015-03,Vodafone,mobile 764,25/02/15 13:15,356,call,2015-03,Three,mobile 765,25/02/15 13:21,1,sms,2015-03,Tesco,mobile 766,25/02/15 13:22,1,sms,2015-03,Tesco,mobile 767,25/02/15 13:22,1,sms,2015-03,Tesco,mobile 768,25/02/15 13:22,1,sms,2015-03,Tesco,mobile 769,25/02/15 13:26,194,call,2015-03,Tesco,mobile 770,25/02/15 13:46,229,call,2015-03,Tesco,mobile 771,25/02/15 15:45,32,call,2015-03,Vodafone,mobile 772,26/02/15 06:58,34.429,data,2015-03,data,data 773,26/02/15 16:38,570,call,2015-03,Vodafone,mobile 774,26/02/15 22:34,29,call,2015-03,voicemail,voicemail 775,27/02/15 06:58,34.429,data,2015-03,data,data 776,27/02/15 13:50,27,call,2015-03,Three,mobile 777,27/02/15 14:33,107,call,2015-03,Three,mobile 778,27/02/15 14:36,128,call,2015-03,Vodafone,mobile 779,28/02/15 06:58,34.429,data,2015-03,data,data 780,28/02/15 14:09,335,call,2015-03,landline,landline 781,28/02/15 14:57,49,call,2015-03,Meteor,mobile 782,28/02/15 15:09,5,call,2015-03,Three,mobile 783,28/02/15 15:49,305,call,2015-03,landline,landline 784,28/02/15 16:00,207,call,2015-03,landline,landline 785,28/02/15 17:13,11,call,2015-03,landline,landline 786,28/02/15 17:14,3,call,2015-03,landline,landline 787,28/02/15 17:17,33,call,2015-03,landline,landline 788,28/02/15 21:25,357,call,2015-03,Three,mobile 789,28/02/15 21:55,1,sms,2015-03,Vodafone,mobile 790,28/02/15 22:39,1,sms,2015-03,Three,mobile 791,01/03/15 06:58,34.429,data,2015-03,data,data 792,01/03/15 12:19,9,call,2015-03,Meteor,mobile 793,02/03/15 06:58,34.429,data,2015-03,data,data 794,02/03/15 09:19,1,sms,2015-03,Vodafone,mobile 795,02/03/15 09:19,1,sms,2015-03,Vodafone,mobile 796,02/03/15 09:23,1,sms,2015-03,Vodafone,mobile 797,02/03/15 09:30,1,sms,2015-03,Vodafone,mobile 798,02/03/15 09:30,1,sms,2015-03,Vodafone,mobile 799,02/03/15 13:07,463,call,2015-03,Three,mobile 800,02/03/15 14:53,2,call,2015-03,voicemail,voicemail 801,02/03/15 14:54,93,call,2015-03,voicemail,voicemail 802,02/03/15 17:35,192,call,2015-03,Meteor,mobile 803,02/03/15 20:48,34,call,2015-03,Tesco,mobile 804,03/03/15 06:58,34.429,data,2015-03,data,data 805,03/03/15 09:57,76,call,2015-03,landline,landline 806,03/03/15 09:59,355,call,2015-03,Three,mobile 807,03/03/15 10:12,745,call,2015-03,Vodafone,mobile 808,03/03/15 10:27,57,call,2015-03,Vodafone,mobile 809,03/03/15 14:34,1325,call,2015-03,Vodafone,mobile 810,03/03/15 18:36,768,call,2015-03,Three,mobile 811,04/03/15 06:58,34.429,data,2015-03,data,data 812,04/03/15 07:02,1,sms,2015-03,Vodafone,mobile 813,04/03/15 07:16,1,sms,2015-03,Vodafone,mobile 814,04/03/15 10:30,1,sms,2015-03,Three,mobile 815,04/03/15 10:30,1,sms,2015-03,Three,mobile 816,04/03/15 12:29,10528,call,2015-03,landline,landline 817,05/03/15 06:58,34.429,data,2015-03,data,data 818,06/03/15 06:58,34.429,data,2015-03,data,data 819,07/03/15 06:58,34.429,data,2015-03,data,data 820,08/03/15 06:58,34.429,data,2015-03,data,data 821,09/03/15 06:58,34.429,data,2015-03,data,data 822,10/03/15 06:58,34.429,data,2015-03,data,data 823,11/03/15 06:58,34.429,data,2015-03,data,data 824,12/03/15 06:58,34.429,data,2015-03,data,data 825,13/03/15 00:38,1,sms,2015-03,world,world 826,13/03/15 00:39,1,sms,2015-03,Vodafone,mobile 827,13/03/15 06:58,34.429,data,2015-03,data,data 828,14/03/15 00:13,1,sms,2015-03,world,world 829,14/03/15 00:16,1,sms,2015-03,world,world ================================================ FILE: utils_test.go ================================================ package datatable_test import ( "testing" "github.com/datasweet/datatable" "github.com/stretchr/testify/assert" ) // checkTable to check if a table contains cells func checkTable(t *testing.T, tb *datatable.DataTable, cells ...interface{}) { ncols := tb.NumCols() nrows := tb.NumRows() assert.Len(t, cells, ncols*(nrows+1)) // + headers cols := tb.Columns() rows := tb.Rows() for i, v := range cells { r := i/ncols - 1 c := i % ncols if r == -1 { assert.Equal(t, v, cols[c], "HEADER COL #%d", r, c) continue } assert.Equal(t, v, rows[r][cols[c]], "ROW #%d, COL #%d", r, c) } } func New(t *testing.T) *datatable.DataTable { tb := datatable.New("test") tb.AddColumn("champ", datatable.String, datatable.Values("Malzahar", "Xerath", "Teemo")) tb.AddColumn("champion", datatable.String, datatable.Expr("upper(`champ`)")) tb.AddColumn("win", datatable.Int, datatable.Values(10, 20, 666)) tb.AddColumn("loose", datatable.Int, datatable.Values(6, 5, 666)) tb.AddColumn("winRate", datatable.String, datatable.Expr("(`win` * 100 / (`win` + `loose`)) ~ \" %\"")) tb.AddColumn("sum", datatable.Float64, datatable.Expr("sum(`win`)")) tb.AddColumn("ok", datatable.Bool, datatable.Expr("true")) tb.AddColumn("hidden", datatable.Bool, datatable.Expr("false")) tb.HideColumn("hidden") checkTable(t, tb, "champ", "champion", "win", "loose", "winRate", "sum", "ok", "Malzahar", "MALZAHAR", 10, 6, "62.5 %", 696.0, true, "Xerath", "XERATH", 20, 5, "80 %", 696.0, true, "Teemo", "TEEMO", 666, 666, "50 %", 696.0, true, ) return tb } ================================================ FILE: where.go ================================================ package datatable // Where filters the datatable based on a predicate func (t *DataTable) Where(predicate func(row Row) bool) *DataTable { if predicate == nil { return t.EmptyCopy() } if err := t.evaluateExpressions(); err != nil { panic(err) } subset := make([]int, 0, t.nrows) // max for i := 0; i < t.nrows; i++ { r := make(Row, len(t.cols)) for _, col := range t.cols { r[col.name] = col.serie.Get(i) } if predicate(r) { subset = append(subset, i) } } cpy := t.EmptyCopy() if len(subset) == 0 { return cpy } cpy.nrows = len(subset) for i, col := range t.cols { cpy.cols[i].serie = col.serie.Pick(subset...) } return cpy } ================================================ FILE: where_test.go ================================================ package datatable_test import ( "strings" "testing" "time" "github.com/datasweet/cast" "github.com/datasweet/datatable" "github.com/stretchr/testify/assert" ) func TestWhere(t *testing.T) { // from join test customers, orders := sampleForJoin() dt, err := customers.LeftJoin(orders, datatable.On("[Customers].[id]", "[Orders].[user_id]")) assert.NoError(t, err) assert.NotNil(t, dt) checkTable(t, dt, "id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 2, "Esmée", "Lefort", "esmee.lefort@example.com", "Lyon", time.Date(2013, time.February, 17, 0, 0, 0, 0, time.UTC), "A00105", 149.45, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, 4, "Luc", "Rolland", "lucrolland@example.com", "Marseille", nil, nil, nil, ) dt = dt.Where(func(row datatable.Row) bool { prenom, okp := cast.AsString(row["prenom"]) num_facture, okf := cast.AsString(row["num_facture"]) return okp && okf && (strings.ToLower(prenom) == "aimée" || num_facture == "A00106") }) checkTable(t, dt, "id", "prenom", "nom", "email", "ville", "date_achat", "num_facture", "prix_total", 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.January, 23, 0, 0, 0, 0, time.UTC), "A00103", 203.14, 1, "Aimée", "Marechal", "aime.marechal@example.com", "Paris", time.Date(2013, time.February, 14, 0, 0, 0, 0, time.UTC), "A00104", 124.00, 3, "Marine", "Prevost", "m.prevost@example.com", "Lille", time.Date(2013, time.February, 21, 0, 0, 0, 0, time.UTC), "A00106", 235.35, ) }