[config] [dsl] fixes #32 - RXT scraper definition language

.gitignore

@@ -18,3 +18,13 @@
 
 # screen logs
 screenlog.*
+
+# Temp files
+*.swo
+*.swp
+*.orig
+
+
+**/mocks/*
+!**/mocks/.gitkeep
+

.golangci.yml

@@ -24,13 +24,14 @@ run:
   # from this option's value:
   #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
   skip-dirs:
+    - src/core/mocks
 
   # which files to skip: they will be analyzed, but issues from them
   # won't be reported. Default value is empty list, but there is
   # no need to include all autogenerated files, we confidently recognize
   # autogenerated files. If it's not please let us know.
   skip-files:
-
+    - src/rxt/grammar/lexer.nn.go
 
 # output configuration options
 output:

.travis.yml

@@ -18,6 +18,7 @@ install:
   - if [[ ! -d $GOPATH/src/github.com/simelo/rextporter ]]; then mkdir -p $GOPATH/src/github.com/simelo; ln -s $TRAVIS_BUILD_DIR $GOPATH/src/github.com/simelo/rextporter; fi
   - cd $GOPATH/src/github.com/simelo/rextporter
   - go get -t ./...
+  - go get github.com/vektra/mockery/.../
   - make install-linters
 
 

Gopkg.toml

@@ -56,7 +56,3 @@
 [[constraint]]
   branch = "master"
   name = "github.com/cznic/goyacc"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/cznic/goyacc"

Makefile

@@ -1,9 +1,21 @@
 .DEFAULT_GOAL := help
 .PHONY: test install-linters test-386 test-amd64 lint
+
+build-grammar: ## Generate source code for REXT grammar
+	nex -s src/rxt/grammar/lexer.nex
 
-test: ## Run test with GOARCH=Default
+mocks: ## Create all mock files for unit tests
+	echo "Generating mock files"
+	cd src/core/ ; mockery -all ; cd ../../
+
+test-grammar: build-grammar ## Test cases for REXT lexer and parser
+	go run cmd/rxtc/lexer.go < src/rxt/testdata/skyexample.rxt 2> src/rxt/testdata/skyexample.golden.orig
+	diff -u src/rxt/testdata/skyexample.golden src/rxt/testdata/skyexample.golden.orig
+
+test: mocks ## Run test with GOARCH=Default
 	go test -count=1 github.com/simelo/rextporter/src/config
 	go test -count=1 github.com/simelo/rextporter/src/scrapper
+	go test -count=1 github.com/simelo/rextporter/src/memconfig
 	if ! screen -list | grep -q "fakeSkycoinForIntegrationTest"; then echo "creating screen fakeSkycoinForIntegrationTest"; screen -L -dm -S fakeSkycoinForIntegrationTest go run test/integration/fake_skycoin_node.go; else echo "fakeSkycoinForIntegrationTest screen already exist. quiting it to create a new one"; screen -S fakeSkycoinForIntegrationTest -X quit; screen -dm -S fakeSkycoinForIntegrationTest go run test/integration/fake_skycoin_node.go; fi
 	sleep 3
 	go test -count=1 -cpu=1 -parallel=1 github.com/simelo/rextporter/test/integration -args -test.v
@@ -13,9 +25,10 @@ test: ## Run test with GOARCH=Default
 	cat screenlog.0
 
 
-test-386: ## Run tests  with GOARCH=386
+test-386: mocks ## Run tests  with GOARCH=386
 	GOARCH=386 go test -count=1 github.com/simelo/rextporter/src/config
 	GOARCH=386 go test -count=1 github.com/simelo/rextporter/src/scrapper
+	GOARCH=386 go test -count=1 github.com/simelo/rextporter/src/memconfig
 	if ! screen -list | grep -q "fakeSkycoinForIntegrationTest"; then echo "creating screen fakeSkycoinForIntegrationTest"; screen -L -dm -S fakeSkycoinForIntegrationTest go run test/integration/fake_skycoin_node.go; else echo "fakeSkycoinForIntegrationTest screen already exist. quiting it to create a new one"; screen -S fakeSkycoinForIntegrationTest -X quit; screen -dm -S fakeSkycoinForIntegrationTest go run test/integration/fake_skycoin_node.go; fi
 	sleep 3
 	GOARCH=386 go test -cpu=1 -parallel=1  -count=1 github.com/simelo/rextporter/test/integration -args -test.v
@@ -24,9 +37,10 @@ test-386: ## Run tests  with GOARCH=386
 	screen -S fakeSkycoinForIntegrationTest -X quit
 	cat screenlog.0
 
-test-amd64: ## Run tests with GOARCH=amd64
+test-amd64: mocks ## Run tests with GOARCH=amd64
 	GOARCH=amd64 go test -count=1 github.com/simelo/rextporter/src/config
 	GOARCH=amd64 go test -count=1 github.com/simelo/rextporter/src/scrapper
+	GOARCH=amd64 go test -count=1 github.com/simelo/rextporter/src/memconfig
 	if ! screen -list | grep -q "fakeSkycoinForIntegrationTest"; then echo "creating screen fakeSkycoinForIntegrationTest"; screen -L -dm -S fakeSkycoinForIntegrationTest go run test/integration/fake_skycoin_node.go; else echo "fakeSkycoinForIntegrationTest screen already exist. quiting it to create a new one"; screen -S fakeSkycoinForIntegrationTest -X quit; screen -dm -S fakeSkycoinForIntegrationTest go run test/integration/fake_skycoin_node.go; fi
 	sleep 3
 	GOARCH=amd64 go test -cpu=1 -parallel=1  -count=1 github.com/simelo/rextporter/test/integration -args -test.v

cmd/rextporter/main.go

@@ -2,20 +2,35 @@ package main
 
 import (
 	"flag"
+	"os"
 
-	"github.com/simelo/rextporter/src/config"
+	"github.com/simelo/rextporter/src/core"
 	"github.com/simelo/rextporter/src/exporter"
+	"github.com/simelo/rextporter/src/toml2config"
+	"github.com/simelo/rextporter/src/tomlconfig"
+	log "github.com/sirupsen/logrus"
 )
 
 func main() {
-	mainConfigFile := flag.String("config", "", "Metrics main config file path.")
+	// log.SetFlags(log.LstdFlags | log.Lshortfile)
+	mainConfigFile := flag.String("config", "/home/adacosta/.config/simelo/rextporter/main.toml", "Metrics main config file path.")
 	defaultListenPort := 8080
 	listenPort := flag.Uint("port", uint(defaultListenPort), "Listen port.")
 	defaultHandlerEndpoint := "/metrics"
 	handlerEndpoint := flag.String("handler", defaultHandlerEndpoint, "Handler endpoint.")
 	flag.Parse()
-	conf := config.MustConfigFromFileSystem(*mainConfigFile)
-	exporter.MustExportMetrics(*handlerEndpoint, uint16(*listenPort), conf)
+	conf, err := tomlconfig.ReadConfigFromFileSystem(*mainConfigFile)
+	if err != nil {
+		log.WithError(err).Errorln("error reading config from file system")
+		os.Exit(1)
+	}
+	var rootConf core.RextRoot
+	rootConf, err = toml2config.Fill(conf)
+	if err != nil {
+		log.WithError(err).Errorln("error filling config info")
+		os.Exit(1)
+	}
+	exporter.MustExportMetrics(*handlerEndpoint, uint16(*listenPort), rootConf)
 	waitForEver := make(chan bool)
 	<-waitForEver
 }

cmd/rxtc/lexer.go

@@ -0,0 +1,10 @@
+package main
+
+import (
+	"github.com/simelo/rextporter/src/rxt"
+	"github.com/simelo/rextporter/src/rxt/grammar"
+)
+
+func main() {
+	grammar.LexTheRxt(&rxt.TokenWriter{}, "LEX")
+}

src/client/api_rest.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/oliveagle/jsonpath"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/simelo/rextporter/src/config"
+	"github.com/simelo/rextporter/src/core"
 	"github.com/simelo/rextporter/src/util"
 	log "github.com/sirupsen/logrus"
 )
@@ -26,19 +26,59 @@ type APIRestCreator struct {
 }
 
 // CreateAPIRestCreator create an APIRestCreator
-func CreateAPIRestCreator(metric config.Metric, service config.Service, dataSourceResponseDurationDesc *prometheus.Desc) (cf CacheableFactory, err error) {
+func CreateAPIRestCreator(resConf core.RextResourceDef, srvConf core.RextServiceDef, dataSourceResponseDurationDesc *prometheus.Desc) (cf CacheableFactory, err error) {
+	resOptions := resConf.GetOptions()
+	httpMethod, err := resOptions.GetString(core.OptKeyRextResourceDefHTTPMethod)
+	if err != nil {
+		log.WithError(err).Errorln("Can not find httpMethod")
+		return cf, err
+	}
+	resURI := strings.TrimPrefix(resConf.GetResourcePATH(srvConf.GetBasePath()), srvConf.GetBasePath())
+	auth := resConf.GetAuth(srvConf.GetAuthForBaseURL())
+	var tkHeaderKey, tkKeyFromEndpoint, tkKeyGenEndpoint string
+	if auth != nil {
+		authOpts := auth.GetOptions()
+		tkHeaderKey, err = authOpts.GetString(core.OptKeyRextAuthDefTokenHeaderKey)
+		if err != nil {
+			log.WithError(err).Errorln("Can not find tokenHeaderKey")
+			return cf, err
+		}
+		tkKeyFromEndpoint, err = authOpts.GetString(core.OptKeyRextAuthDefTokenKeyFromEndpoint)
+		if err != nil {
+			log.WithError(err).Errorln("Can not find tokenKeyFromEndpoint")
+			return cf, err
+		}
+		tkKeyGenEndpoint, err = authOpts.GetString(core.OptKeyRextAuthDefTokenGenEndpoint)
+		if err != nil {
+			log.WithError(err).Errorln("Can not find tkKeyGenEndpoint")
+			return cf, err
+		}
+	} else {
+		log.Warnln("you have an empty auth")
+	}
+	srvOpts := srvConf.GetOptions()
+	jobName, err := srvOpts.GetString(core.OptKeyRextServiceDefJobName)
+	if err != nil {
+		log.WithError(err).Errorln("Can not find jobName")
+		return cf, err
+	}
+	instanceName, err := srvOpts.GetString(core.OptKeyRextServiceDefInstanceName)
+	if err != nil {
+		log.WithError(err).Errorln("Can not find instanceName")
+		return cf, err
+	}
 	cf = APIRestCreator{
 		baseFactory: baseFactory{
-			jobName:                        service.JobName(),
-			instanceName:                   service.InstanceName(),
-			dataSource:                     metric.URL,
+			jobName:                        jobName,
+			instanceName:                   instanceName,
+			dataSource:                     resURI,
 			dataSourceResponseDurationDesc: dataSourceResponseDurationDesc,
 		},
-		httpMethod:           metric.HTTPMethod,
-		dataPath:             service.URIToGetMetric(metric),
-		tokenPath:            service.URIToGetToken(),
-		tokenHeaderKey:       service.TokenHeaderKey,
-		tokenKeyFromEndpoint: service.TokenKeyFromEndpoint,
+		httpMethod:           httpMethod,
+		dataPath:             resConf.GetResourcePATH(srvConf.GetBasePath()),
+		tokenPath:            tkKeyGenEndpoint,
+		tokenHeaderKey:       tkHeaderKey,
+		tokenKeyFromEndpoint: tkKeyFromEndpoint,
 	}
 	return cf, err
 }

src/client/metrics_forwader.go

@@ -2,16 +2,16 @@ package client
 
 import (
 	"compress/gzip"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"time"
 
-	"github.com/simelo/rextporter/src/config"
+	"github.com/simelo/rextporter/src/core"
 	"github.com/simelo/rextporter/src/util"
 	"github.com/simelo/rextporter/src/util/metrics"
+	log "github.com/sirupsen/logrus"
 )
 
 // ProxyMetricClientCreator create a metrics fordwader client
@@ -23,15 +23,24 @@ type ProxyMetricClientCreator struct {
 }
 
 // CreateProxyMetricClientCreator create a ProxyMetricClientCreator with required info to create a metrics fordwader client
-func CreateProxyMetricClientCreator(service config.Service, fDefMetrics *metrics.DefaultFordwaderMetrics) (cf ProxyMetricClientCreator, err error) {
-	if !util.StrSliceContains(service.Modes, config.ServiceTypeProxy) {
-		return ProxyMetricClientCreator{}, errors.New("can not create a forward_metrics metric client from a service whitout type " + config.ServiceTypeProxy)
+func CreateProxyMetricClientCreator(resConf core.RextResourceDef, srvConf core.RextServiceDef, fDefMetrics *metrics.DefaultFordwaderMetrics) (cf ProxyMetricClientCreator, err error) {
+	srvOpts := srvConf.GetOptions()
+	jobName, err := srvOpts.GetString(core.OptKeyRextServiceDefJobName)
+	if err != nil {
+		log.WithError(err).Errorln("Can not find jobName")
+		return cf, err
 	}
+	instanceName, err := srvOpts.GetString(core.OptKeyRextServiceDefInstanceName)
+	if err != nil {
+		log.WithError(err).Errorln("Can not find instanceName")
+		return cf, err
+	}
+	resPath := resConf.GetResourcePATH(srvConf.GetBasePath())
 	cf = ProxyMetricClientCreator{
 		defFordwaderMetrics: fDefMetrics,
-		dataPath:            service.URIToGetExposedMetric(),
-		JobName:             service.JobName(),
-		InstanceName:        service.InstanceName(),
+		dataPath:            resPath,
+		JobName:             jobName,
+		InstanceName:        instanceName,
 	}
 	return cf, err
 }