Merge pull request RediSearch#11 from filipecosta90/go.mod

Revamped project. Enabled go modules. Checking format.

Author: filipecosta90

.gitignore

@@ -1,3 +1,8 @@
 *.a
 *.o
 RediSearchBenchmark
+bin/*
+*.csv
+*.xml
+*.gz
+.idea/*

Makefile

@@ -0,0 +1,43 @@
+# Go parameters
+GOCMD=GO111MODULE=on go
+GOBUILD=$(GOCMD) build
+GOINSTALL=$(GOCMD) install
+GOCLEAN=$(GOCMD) clean
+GOTEST=$(GOCMD) test
+GOGET=$(GOCMD) get -v
+GOMOD=$(GOCMD) mod
+GOFMT=$(GOCMD) fmt
+
+# Build time variables
+ifeq ($(GIT_SHA),)
+GIT_SHA:=$(shell git rev-parse HEAD)
+endif
+
+ifeq ($(GIT_DIRTY),)
+GIT_DIRTY:=$(shell git diff --no-ext-diff 2> /dev/null | wc -l)
+endif
+
+.PHONY: all
+
+all: get RediSearchBenchmark
+fmt:
+	$(GOFMT) ./...
+
+
+checkfmt:
+	@echo 'Checking gofmt';\
+ 	bash -c "diff -u <(echo -n) <(gofmt -d .)";\
+	EXIT_CODE=$$?;\
+	if [ "$$EXIT_CODE"  -ne 0 ]; then \
+		echo '$@: Go files must be formatted with gofmt'; \
+	fi && \
+	exit $$EXIT_CODE
+
+get:
+	$(GOGET) -t -v ./...
+
+test: get fmt
+	$(GOTEST) -v -race -coverprofile=coverage.txt -covermode=atomic ./...
+
+RediSearchBenchmark: $(wildcard ./index/*.go) $(wildcard ./ingest/*.go) $(wildcard ./query/*.go) $(wildcard ./synth/*.go) 
+	$(GOBUILD) -o ./bin/$@ -ldflags="-X 'main.GitSHA1=$(GIT_SHA)' -X 'main.GitDirty=$(GIT_DIRTY)'" .

README.md

@@ -3,11 +3,16 @@
 [![GoDoc](https://godoc.org/github.com/RediSearch/RediSearchBenchmark?status.svg)](https://godoc.org/github.com/RediSearch/RediSearchBenchmark)
 
 # RediSearchBenchmarks
-This is a Go application (originally written by Dvir Volk) which supports reading, indexing and searching in [Wikipedia Abstract Data Dumps](https://s3.amazonaws.com/uploads-files/wiki_dump.gz) using two search engines:  
+This is a Go application (originally written by Dvir Volk) which supports reading, indexing and searching using two search engines:
 
-* [RediSearch](https://github.com/RedisLabsModules/RediSearch)
+* [RediSearch](https://github.com/RediSearch/RediSearch)
 * [Elasticsearch](https://www.elastic.co/)
 
+with the following datasets:
+
+* [Wikipedia Abstract Data Dumps](https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/enwiki-abstract/enwiki-latest-abstract.xml): from English-language Wikipedia:Database page abstracts. This use case generates 3 TEXT fields per document.
+* [pmc](https://s3.amazonaws.com/benchmarks.redislabs/redisearch/datasets/pmc/documents.json): Full text benchmark with academic papers from PMC.
+
 ## Usage 
 
 ```

go.mod

@@ -0,0 +1,13 @@
+module github.com/RediSearch/RediSearchBenchmark
+
+go 1.16
+
+require (
+	github.com/garyburd/redigo v1.6.2
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/olivere/elastic v6.2.35+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/stretchr/testify v1.7.0
+	github.com/vanng822/go-solr v0.10.0
+	gopkg.in/olivere/elastic.v6 v6.2.35
+)

go.sum

@@ -0,0 +1,24 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/garyburd/redigo v1.6.2 h1:yE/pwKCrbLpLpQICzYTeZ7JsTA/C53wFTJHaEtRqniM=
+github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/olivere/elastic v6.2.35+incompatible h1:MMklYDy2ySi01s123CB2WLBuDMzFX4qhFcA5tKWJPgM=
+github.com/olivere/elastic v6.2.35+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/vanng822/go-solr v0.10.0 h1:oygAxyFL2apSN8vddxDXoyho40z66Guu9nwH7ANr6wM=
+github.com/vanng822/go-solr v0.10.0/go.mod h1:FSglzTPzoNVKTXP+SqEQiiz284cKzcKpeRXmwPa81wc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/olivere/elastic.v6 v6.2.35 h1:/5dJ0UzM231DGl9eDYOdDgv8yCFzAHQVPMo69rnylks=
+gopkg.in/olivere/elastic.v6 v6.2.35/go.mod h1:2cTT8Z+/LcArSWpCgvZqBgt3VOqXiy7v00w12Lz8bd4=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

index/elastic/elastic.go

@@ -6,19 +6,19 @@ import (
 	"net/http"
 	"time"
 
+	"context"
 	"github.com/RediSearch/RediSearchBenchmark/index"
 	"github.com/RediSearch/RediSearchBenchmark/query"
 	"gopkg.in/olivere/elastic.v6"
-	"context"
 )
 
 // Index is an ElasticSearch index
 type Index struct {
 	conn *elastic.Client
 
-	md   *index.Metadata
-	name string
-	typ  string
+	md           *index.Metadata
+	name         string
+	typ          string
 	disableCache bool
 }
 
@@ -27,7 +27,7 @@ var conn *elastic.Client = nil
 // NewIndex creates a new elasticSearch index with the given address and name. typ is the entity type
 func NewIndex(addr, name, typ string, disableCache bool, md *index.Metadata) (*Index, error) {
 	var err error
-	if conn == nil{
+	if conn == nil {
 		client := &http.Client{
 			Transport: &http.Transport{
 				MaxIdleConnsPerHost: 200,
@@ -39,12 +39,12 @@ func NewIndex(addr, name, typ string, disableCache bool, md *index.Metadata) (*I
 			return nil, err
 		}
 	}
-	
+
 	ret := &Index{
-		conn: conn,
-		md:   md,
-		name: name,
-		typ:  typ,
+		conn:         conn,
+		md:           md,
+		name:         name,
+		typ:          typ,
 		disableCache: disableCache,
 	}
 
@@ -95,10 +95,10 @@ func (i *Index) Create() error {
 	// 			"payloads": true,
 	// 		},
 	// 	},
-	// }	
+	// }
 
 	mappings := map[string]mapping{
-		i.typ:          doc,
+		i.typ: doc,
 		// "autocomplete": ac,
 	}
 
@@ -127,7 +127,7 @@ func (i *Index) Index(docs []index.Document, opts interface{}) error {
 	}
 	_, err := blk.Refresh("true").Do(context.Background())
 
-	if err != nil{
+	if err != nil {
 		panic(err)
 	}
 

index/redisearch/index.go

@@ -36,7 +36,7 @@ type IndexingOptions struct {
 
 type ConnectionPool struct {
 	sync.Mutex
-	pools		   map[string]*redis.Pool
+	pools map[string]*redis.Pool
 }
 
 var connectionPool = ConnectionPool{
@@ -48,7 +48,7 @@ type Index struct {
 	sync.Mutex
 	hosts         []string
 	password      string
-	temporary	  int
+	temporary     int
 	md            *index.Metadata
 	name          string
 	commandPrefix string
@@ -64,12 +64,12 @@ func (i *Index) getConn() redis.Conn {
 	if !found {
 		pool = redis.NewPool(func() (redis.Conn, error) {
 			// TODO: Add timeouts. and 2 separate pools for indexing and querying, with different timeouts
-			if i.password != ""{
-				return redis.Dial("tcp", host, redis.DialPassword(i.password))	
-			}else{
-				return redis.Dial("tcp", host)	
+			if i.password != "" {
+				return redis.Dial("tcp", host, redis.DialPassword(i.password))
+			} else {
+				return redis.Dial("tcp", host)
 			}
-			
+
 		}, maxConns)
 		pool.TestOnBorrow = func(c redis.Conn, t time.Time) error {
 			if time.Since(t).Seconds() > 3 {
@@ -92,8 +92,8 @@ func NewIndex(addrs []string, pass string, temporary int, name string, md *index
 
 		hosts: addrs,
 
-		md: md,
-		password: pass,
+		md:        md,
+		password:  pass,
 		temporary: temporary,
 
 		name: name,
@@ -121,7 +121,7 @@ func (i *Index) GetName() string {
 func (i *Index) Create() error {
 
 	args := redis.Args{i.name}
-	if i.temporary != -1{
+	if i.temporary != -1 {
 		t := strconv.Itoa(i.temporary)
 		args = append(args, "TEMPORARY", t)
 	}

ingest/ingest.go

@@ -190,16 +190,16 @@ func ReadFile(fileName string, r DocumentReader, idx index.Index, ac index.Autoc
 			for doc := range doch {
 				if doc.Id != "" {
 					docs = append(docs, doc)
-					numOfDocs++;
-				}else{
+					numOfDocs++
+				} else {
 					fmt.Println("warning empty id")
 				}
-				if(len(docs) > 1000){
+				if len(docs) > 1000 {
 					idx.Index(docs, opts)
 					docs = []index.Document{}
 				}
 			}
-			if(len(docs) > 0){
+			if len(docs) > 0 {
 				idx.Index(docs, opts)
 			}
 		}(doch)

ingest/wikipedia_abstract.go

@@ -86,7 +86,7 @@ func (r *WikipediaAbstractsReader) LoadScores(fileName string) error {
 	return nil
 }
 
-func (wr *WikipediaAbstractsReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index)  error {
+func (wr *WikipediaAbstractsReader) Read(r io.Reader, ch chan index.Document, maxDocsToRead int, idx index.Index) error {
 
 	dec := xml.NewDecoder(r)
 	go func() {
@@ -129,9 +129,9 @@ func (wr *WikipediaAbstractsReader) Read(r io.Reader, ch chan index.Document, ma
 				}
 				currentText = ""
 			}
-			if maxDocsToRead != -1 && docsRead >= maxDocsToRead{
+			if maxDocsToRead != -1 && docsRead >= maxDocsToRead {
 				break
-			} 
+			}
 			tok, err = dec.RawToken()
 
 		}

main.go

@@ -4,15 +4,14 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
-	"strconv"
 
 	"runtime"
 
 	"sync"
 
-
 	"github.com/RediSearch/RediSearchBenchmark/index"
 	"github.com/RediSearch/RediSearchBenchmark/index/elastic"
 	"github.com/RediSearch/RediSearchBenchmark/index/redisearch"
@@ -67,16 +66,16 @@ func main() {
 
 	//scoreFile := flag.String("scores", "", "read scores of documents CSV for indexing")
 	engine := flag.String("engine", "redis", "The search backend to run")
-	benchmark := flag.String("benchmark", "", "[search|suggest] - if set, we run the given benchmark")
+	benchmark := flag.String("benchmark", "", "[search|suggest|wildcard|prefix] - if set, we run the given benchmark")
 	random := flag.Int("random", 0, "Generate random documents with terms like term0..term{N}")
 	indexesAmount := flag.Int("indexes", 1, "number of indexes to generate")
 	// fuzzy := flag.Bool("fuzzy", false, "For redis only - benchmark fuzzy auto suggest")
 	disableCache := flag.Bool("disableCache", false, "for elastic only, disabling query cache")
-	seconds := flag.Int("duration", 5, "number of seconds to run the benchmark")
+	seconds := flag.Int("duration", 60, "number of seconds to run the benchmark")
 	temporary := flag.Int("temporary", -1, "for redisearch only, create a temporary index that will expire after the given amount of seconds, -1 mean no temporary")
 	conc := flag.Int("c", 4, "benchmark concurrency")
 	maxDocPerIndex := flag.Int("maxdocs", -1, "specify the numebr of max docs per index, -1 for no limit")
-	qs := flag.String("queries", "hello world", "comma separated list of queries to benchmark")
+	qs := flag.String("queries", "barack obama", "comma separated list of queries to benchmark. Use this option only for the historical reasons via `-queries='barack obama'`. If you don't specify a value it will read the input file and randomize the input search terms")
 	outfile := flag.String("o", "benchmark.csv", "results output file. set to - for stdout")
 	cmdPrefix := flag.String("prefix", "FT", "Command prefix for FT module")
 	password := flag.String("password", "", "redis database password")
@@ -91,7 +90,7 @@ func main() {
 
 	indexes := make([]index.Index, *indexesAmount)
 	var opts interface{}
-	if *engine == "redis"{
+	if *engine == "redis" {
 		opts = query.QueryVerbatim
 	}
 	// select index to run
@@ -103,7 +102,7 @@ func main() {
 
 	// Search benchmark
 	if *benchmark == "search" {
-		if(*indexesAmount > 1){
+		if *indexesAmount > 1 {
 			panic("search not supported on multiple indexes!!!")
 		}
 		name := fmt.Sprintf("search: %s", *qs)
@@ -122,7 +121,7 @@ func main() {
 	if *random > 0 {
 		indexes[0].Drop()
 		err := indexes[0].Create()
-		if err != nil{
+		if err != nil {
 			panic(err)
 		}
 
@@ -132,7 +131,7 @@ func main() {
 		n := 0
 		ch := make(chan index.Document, N)
 		go func() {
-			for i :=0 ; i < *maxDocPerIndex || *maxDocPerIndex == -1 ; i++{
+			for i := 0; i < *maxDocPerIndex || *maxDocPerIndex == -1; i++ {
 				ch <- gen.Generate(0)
 			}
 		}()
@@ -153,14 +152,14 @@ func main() {
 
 		var wg sync.WaitGroup
 		idxChan := make(chan index.Index, 1)
-		for i := 0 ; i < 30 ; i++{
+		for i := 0; i < 30; i++ {
 			wg.Add(1)
-			go func(idxChan chan index.Index){
+			go func(idxChan chan index.Index) {
 				defer wg.Done()
-				for idx := range idxChan{
+				for idx := range idxChan {
 					idx.Drop()
 					err := idx.Create()
-					if err != nil{
+					if err != nil {
 						panic(err)
 					}
 					wr := &ingest.WikipediaAbstractsReader{}
@@ -179,7 +178,7 @@ func main() {
 			}(idxChan)
 		}
 
-		for _,idx := range indexes{
+		for _, idx := range indexes {
 			idxChan <- idx
 		}
 		close(idxChan)