Skip to content

Commit

Permalink
Add Magefile for easy testing (#37)
Browse files Browse the repository at this point in the history
* Format file

* Ignore idea IDE files

* Add Magefile.go and update README.md

* Fix vet

* - Remove lint
- Add vet to default target
- Add alias for test with race

* Rename check to build

* PR changes resolved

Co-authored-by: Shrivardhan Rao <[email protected]>
  • Loading branch information
cosmic-chichu and Shrivardhan Rao authored Mar 8, 2021
1 parent b547535 commit e1ad4f9
Show file tree
Hide file tree
Showing 7 changed files with 290 additions and 5 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*.dylib
*.html
.vscode/*
.idea/

# Test binary, built with `go test -c`
*.test
Expand Down
231 changes: 231 additions & 0 deletions Magefile.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,231 @@
//+build mage

package main

import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
"sync"

"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
)

var ldflags = ""

// allow user to override go executable by running as GOEXE=xxx make ... on unix-like systems
var goexe = "go"

// Build is the default that fmt, vet, runs test and builds
var Default = Build

func init() {
if exe := os.Getenv("GOEXE"); exe != "" {
goexe = exe
}

// We want to use Go 1.11 modules even if the source lives inside GOPATH.
// The default is "auto".
os.Setenv("GO111MODULE", "on")
}

// Fmt run gofmt linter
func Fmt() error {
if !isGoLatest() {
return nil
}
pkgs, err := packages()
if err != nil {
return err
}
failed := false
first := true
for _, pkg := range pkgs {
files, err := filepath.Glob(filepath.Join(pkg, "*.go"))
if err != nil {
return nil
}
for _, f := range files {
// gofmt doesn't exit with non-zero when it finds unformatted code
// so we have to explicitly look for output, and if we find any, we
// should fail this target.
s, err := sh.Output("gofmt", "-l", f)
if err != nil {
fmt.Printf("ERROR: running gofmt on %q: %v\n", f, err)
failed = true
}
if s != "" {
if first {
fmt.Println("The following files are not gofmt'ed:")
first = false
}
failed = true
fmt.Println(s)
}
}
}
if failed {
return errors.New("improperly formatted go files")
}
return nil
}

// Vet run go vet linter
func Vet() error {
if err := sh.Run(goexe, "vet", "./..."); err != nil {
return fmt.Errorf("error running go vet: %v", err)
}
return nil
}

// Run tests
func Test() error {
env := map[string]string{"GOFLAGS": testGoFlags()}
return runCmd(env, goexe, "test", "./...", buildFlags(), "-tags", buildTags())
}

// TestRace run tests with race detector
func TestRace() error {
env := map[string]string{"GOFLAGS": testGoFlags()}
return runCmd(env, goexe, "test", "-race", "./...", buildFlags(), "-tags", buildTags())
}

// TestCoverHTML generates test coverage report
func TestCoverHTML() error {
const (
coverAll = "coverage-all.out"
cover = "coverage.out"
)
f, err := os.Create(coverAll)
if err != nil {
return err
}
defer f.Close()
if _, err := f.Write([]byte("mode: count")); err != nil {
return err
}
pkgs, err := packages()
if err != nil {
return err
}
for _, pkg := range pkgs {
if err := sh.Run(goexe, "test", "-coverprofile="+cover, pkg); err != nil {
return err
}
b, err := ioutil.ReadFile(cover)
if err != nil {
if os.IsNotExist(err) {
continue
}
return err
}
idx := bytes.Index(b, []byte{'\n'})
b = b[idx+1:]
if _, err := f.Write(b); err != nil {
return err
}
}
if err := f.Close(); err != nil {
return err
}
return sh.Run(goexe, "tool", "cover", "-html="+coverAll)
}

// Build run linters, vet and tests
func Build() error {
if strings.Contains(runtime.Version(), "1.8") {
// Go 1.8 doesn't play along with go test ./... and /vendor.
// We could fix that, but that would take time.
fmt.Printf("Skip Build on %s\n", runtime.Version())
return nil
}

// TODO: Add lint after fixing errors
mg.Deps(Fmt, Vet, TestRace)
return nil
}

var (
pkgPrefixLen = len("github.com/grafadruid/go-druid")
pkgs []string
pkgsInit sync.Once
)

// testGoFlags returns test flags that need to be set
func testGoFlags() string {
return "-v"
}

func packages() ([]string, error) {
var err error
pkgsInit.Do(func() {
var s string
s, err = sh.Output(goexe, "list", "./...")
if err != nil {
return
}
pkgs = strings.Split(s, "\n")
for i := range pkgs {
pkgs[i] = "." + pkgs[i][pkgPrefixLen:]
}
})
return pkgs, err
}

func buildFlags() []string {
if runtime.GOOS == "windows" {
return []string{"-buildmode", "exe"}
}
return nil
}

func buildTags() string {
return "none"
}

func isGoLatest() bool {
return strings.Contains(runtime.Version(), "1.14")
}

func runCmd(env map[string]string, cmd string, args ...interface{}) error {
if mg.Verbose() {
return runWith(env, cmd, args...)
}
output, err := sh.OutputWith(env, cmd, argsToStrings(args...)...)
if err != nil {
fmt.Fprint(os.Stderr, output)
}

return err
}

func runWith(env map[string]string, cmd string, inArgs ...interface{}) error {
s := argsToStrings(inArgs...)
return sh.RunWith(env, cmd, s...)
}

func argsToStrings(v ...interface{}) []string {
var args []string
for _, arg := range v {
switch v := arg.(type) {
case string:
if v != "" {
args = append(args, v)
}
case []string:
if v != nil {
args = append(args, v...)
}
default:
panic("invalid type")
}
}

return args
}
25 changes: 25 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,28 @@
# go-druid
A Golang client for Druid.
Now supports Query API and Common API.

### Development

#### Testing
`go-druid` uses mage to run tests locally.
Install Mage:
```
git clone https://github.com/magefile/mage
cd mage
go run bootstrap.go
```
`mage -l` provides a list of targets that can be run. Default is `Check`

```
Targets:
build runs go mod download and then installs the binary.
check* run linters and tests
fmt run gofmt linter
lint run golint linter https://github.com/golang/lint
testCoverHTML generates test coverage report
testRace run tests with race detector
vet run go vet linter
* default target
```
2 changes: 1 addition & 1 deletion builder/query/group_by.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type GroupBy struct {
PostAggregations []builder.PostAggregator `json:"postAggregations,omitempty"`
Having builder.HavingSpec `json:"having,omitempty"`
LimitSpec builder.LimitSpec `json:"limitSpec,omitempty"`
SubtotalsSpec [][]string `json:"subtotalsSpec",omitempty`
SubtotalsSpec [][]string `json:"subtotalsSpec,omitempty"`
}

func NewGroupBy() *GroupBy {
Expand Down
5 changes: 1 addition & 4 deletions examples/tdigestExample.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ func getConnection() *druid.Client {
To experiment, you can use the doubles_sketch_data.tsv file attached in this repo. It is a copy of https://github.com/apache/druid/blob/master/extensions-contrib/tdigestsketch/src/test/resources/doubles_sketch_data.tsv
*/


// tdigestSketchUsingBuilder example using Builder Pattern
func tdigestSketchUsingBuilder() {
d := getConnection()
Expand All @@ -49,18 +48,16 @@ func tdigestSketchUsingBuilder() {
atds := aggregation.NewTDigestSketch().SetName("merged_sketch").SetFieldName("valuesTDS")
a := []builder.Aggregator{atds}


//TDigest Post Aggregation
qf := postaggregation.NewQuantilesFromTDigestSketchField().
SetType("fieldAccess").
SetFieldName("merged_sketch")
qa := postaggregation.NewQuantilesFromTDigestSketch().
SetField(qf).
SetFractions([]float64{0.25, 0.5, 0.75, 0.9, 0.95, 0.99}). // add additional quantiles as needed
SetFractions([]float64{0.25, 0.5, 0.75, 0.9, 0.95, 0.99}). // add additional quantiles as needed
SetName("quantiles")
pa := []builder.PostAggregator{qa}


ts := query.NewTimeseries().SetDataSource(table).SetIntervals(is).SetAggregations(a).SetPostAggregations(pa).SetGranularity(m).SetLimit(10)
var results interface{}
_, err := d.Query().Execute(ts, &results)
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ require (
github.com/davecgh/go-spew v1.1.1
github.com/google/go-querystring v1.0.0
github.com/hashicorp/go-retryablehttp v0.6.7
github.com/magefile/mage v1.11.0
github.com/stretchr/testify v1.2.2
)
30 changes: 30 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,37 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-retryablehttp v0.6.7 h1:8/CAEZt/+F7kR7GevNHulKkUjLht3CPmn7egmhieNKo=
github.com/hashicorp/go-retryablehttp v0.6.7/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls=
github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

0 comments on commit e1ad4f9

Please sign in to comment.