Merge branch 'release/2.3.0'

ONSdigital · Apr 20, 2020 · b4e2ba8 · b4e2ba8
2 parents d17a526 + d4dd61e
commit b4e2ba8
Show file tree

Hide file tree

Showing 493 changed files with 1,118 additions and 74,733 deletions.
diff --git a/Makefile b/Makefile
@@ -2,23 +2,24 @@ SHELL=bash
 MAIN=dp-filter-api
 
 BUILD=build
-BUILD_ARCH=$(BUILD)/$(GOOS)-$(GOARCH)
 BIN_DIR?=.
 
-export GOOS?=$(shell go env GOOS)
-export GOARCH?=$(shell go env GOARCH)
+BUILD_TIME=$(shell date +%s)
+GIT_COMMIT=$(shell git rev-parse HEAD)
+VERSION ?= $(shell git tag --points-at HEAD | grep ^v | head -n 1)
+LDFLAGS=-ldflags "-w -s -X 'main.Version=${VERSION}' -X 'main.BuildTime=$(BUILD_TIME)' -X 'main.GitCommit=$(GIT_COMMIT)'"
 
 DATABASE_ADDRESS?=bolt://localhost:7687
 
 build:
-	@mkdir -p $(BUILD_ARCH)/$(BIN_DIR)
-	go build -o $(BUILD_ARCH)/$(BIN_DIR)/dp-filter-api cmd/$(MAIN)/main.go
+	@mkdir -p $(BUILD)/$(BIN_DIR)
+	go build $(LDFLAGS) -o $(BUILD)/$(BIN_DIR)/dp-filter-api cmd/$(MAIN)/main.go
 debug:
-	GRAPH_DRIVER_TYPE=neo4j GRAPH_ADDR="$(DATABASE_ADDRESS)" HUMAN_LOG=1 go run -race cmd/$(MAIN)/main.go
+	GRAPH_DRIVER_TYPE=neo4j GRAPH_ADDR="$(DATABASE_ADDRESS)" HUMAN_LOG=1 go run $(LDFLAGS) -race cmd/$(MAIN)/main.go
 acceptance-publishing:
-	MONGODB_FILTERS_DATABASE=test GRAPH_DRIVER_TYPE=neo4j GRAPH_ADDR="$(DATABASE_ADDRESS)" HUMAN_LOG=1 go run -race cmd/$(MAIN)/main.go
+	MONGODB_FILTERS_DATABASE=test GRAPH_DRIVER_TYPE=neo4j GRAPH_ADDR="$(DATABASE_ADDRESS)" HUMAN_LOG=1 go run $(LDFLAGS) -race cmd/$(MAIN)/main.go
 acceptance-web:
-	ENABLE_PRIVATE_ENDPOINTS=false MONGODB_FILTERS_DATABASE=test GRAPH_DRIVER_TYPE=neo4j GRAPH_ADDR="$(DATABASE_ADDRESS)" HUMAN_LOG=1 go run -race cmd/$(MAIN)/main.go
+	ENABLE_PRIVATE_ENDPOINTS=false MONGODB_FILTERS_DATABASE=test GRAPH_DRIVER_TYPE=neo4j GRAPH_ADDR="$(DATABASE_ADDRESS)" HUMAN_LOG=1 go run $(LDFLAGS) -race cmd/$(MAIN)/main.go
 test:
-	go test -cover $(shell go list ./... | grep -v /vendor/)
+	go test -cover -race ./...
 .PHONY: build debug acceptance test
diff --git a/README.md b/README.md
@@ -19,25 +19,26 @@ Scripts for updating and debugging Kafka can be found [here](https://github.com/
 * Disable authentication in the config
 * Run `brew services start neo4j`
 
-| Environment variable       | Default                                   | Description
-| -------------------------- | ----------------------------------------- | -----------
-| BIND_ADDR                  | :22100                                    | The host and port to bind to
-| HOST                       | http://localhost:22100                    | The host name used to build URLs
-| KAFKA_ADDR                 | localhost:9092                            | The kafka broker addresses (can be comma separated)
-| FILTER_JOB_SUBMITTED_TOPIC | filter-job-submitted                      | The kafka topic to write messages to
-| KAFKA_MAX_BYTES            | 2000000                                   | The maximum permitted size of a message. Should be set equal to or smaller than the broker's `message.max.bytes`
-| MONGODB_BIND_ADDR          | localhost:27017                           | URL to a mongodb services
-| MONGODB_FILTERS_DATABASE   | "filters"                                 | The mongodb database to store filters
-| SHUTDOWN_TIMEOUT           | 5s                                        | The graceful shutdown timeout (`time.Duration` format)
-| DATASET_API_URL            | http://localhost:22000                    | The URL of the Dataset API
-| DATASET_API_AUTH_TOKEN     | FD0108EA-825D-411C-9B1D-41EF7727F465      | The token used to access the Dataset API
-| HEALTHCHECK_INTERVAL       | 30s                                       | Time between self-healthchecks (`time.Duration` format)
-| SERVICE_AUTH_TOKEN         | FD0108EA-825D-411C-9B1D-41EF7727F465      | The token used to identify this service when authenticating
-| ZEBEDEE_URL                | "http://localhost:8082"                   | Zebedee URL
-| ENABLE_PRIVATE_ENDPOINTS   | false                                     | true if private endpoints should be enabled
-| DOWNLOAD_SERVICE_URL       | http://localhost:23600                    | The URL of the download service
-| DOWNLOAD_SERVICE_SECRET_KEY| QB0108EZ-825D-412C-9B1D-41EF7747F462      | The service token for the download service
-| AUDIT_EVENTS_TOPIC         | audit-events                              | The Kafka topic name to send audit events to
+| Environment variable         | Default                                   | Description
+| ---------------------------- | ----------------------------------------- | -----------
+| BIND_ADDR                    | :22100                                    | The host and port to bind to
+| HOST                         | http://localhost:22100                    | The host name used to build URLs
+| KAFKA_ADDR                   | localhost:9092                            | The kafka broker addresses (can be comma separated)
+| FILTER_JOB_SUBMITTED_TOPIC   | filter-job-submitted                      | The kafka topic to write messages to
+| KAFKA_MAX_BYTES              | 2000000                                   | The maximum permitted size of a message. Should be set equal to or smaller than the broker's `message.max.bytes`
+| MONGODB_BIND_ADDR            | localhost:27017                           | URL to a mongodb services
+| MONGODB_FILTERS_DATABASE     | "filters"                                 | The mongodb database to store filters
+| SHUTDOWN_TIMEOUT             | 5s                                        | The graceful shutdown timeout (`time.Duration` format)
+| DATASET_API_URL              | http://localhost:22000                    | The URL of the Dataset API
+| DATASET_API_AUTH_TOKEN       | FD0108EA-825D-411C-9B1D-41EF7727F465      | The token used to access the Dataset API
+| HEALTHCHECK_INTERVAL         | 30s                                       | Time between self-healthchecks (`time.Duration` format)
+| HEALTHCHECK_CRITICAL_TIMEOUT | 90s                                       | The time taken for the health changes from warning state to critical due to subsystem check failures
+| SERVICE_AUTH_TOKEN           | FD0108EA-825D-411C-9B1D-41EF7727F465      | The token used to identify this service when authenticating
+| ZEBEDEE_URL                  | "http://localhost:8082"                   | Zebedee URL
+| ENABLE_PRIVATE_ENDPOINTS     | false                                     | true if private endpoints should be enabled
+| DOWNLOAD_SERVICE_URL         | http://localhost:23600                    | The URL of the download service
+| DOWNLOAD_SERVICE_SECRET_KEY  | QB0108EZ-825D-412C-9B1D-41EF7747F462      | The service token for the download service
+| AUDIT_EVENTS_TOPIC           | audit-events                              | The Kafka topic name to send audit events to
 
 ### Healthchecking
 

diff --git a/api/api.go b/api/api.go
@@ -2,18 +2,17 @@ package api
 
 import (
 	"context"
-
+	"github.com/ONSdigital/dp-api-clients-go/dataset"
 	"github.com/ONSdigital/dp-filter-api/models"
-	"github.com/ONSdigital/dp-filter-api/preview"
+	"github.com/ONSdigital/dp-healthcheck/healthcheck"
 	"github.com/ONSdigital/go-ns/audit"
-	"github.com/ONSdigital/go-ns/clients/dataset"
-	"github.com/ONSdigital/go-ns/healthcheck"
+	"github.com/ONSdigital/go-ns/handlers/collectionID"
 	"github.com/ONSdigital/go-ns/identity"
-	"github.com/ONSdigital/go-ns/log"
 	"github.com/ONSdigital/go-ns/server"
+	"github.com/ONSdigital/log.go/log"
 	"github.com/gorilla/mux"
 	"github.com/justinas/alice"
-	"github.com/ONSdigital/go-ns/handlers/collectionID"
+	"net/http"
 )
 
 //go:generate moq -out datastoretest/preview.go -pkg datastoretest . PreviewDataset
@@ -22,9 +21,9 @@ var httpServer *server.Server
 
 // DatasetAPI - An interface used to access the DatasetAPI
 type DatasetAPI interface {
-	GetVersion(ctx context.Context, id, edition, version string) (m dataset.Version, err error)
-	GetDimensions(ctx context.Context, id, edition, version string) (m dataset.Dimensions, err error)
-	GetOptions(ctx context.Context, id, edition, version, dimension string) (m dataset.Options, err error)
+	GetVersion(ctx context.Context, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, version string) (m dataset.Version, err error)
+	GetVersionDimensions(ctx context.Context, userAuthToken, serviceAuthToken, collectionID, id, edition, version string) (m dataset.VersionDimensions, err error)
+	GetOptions(ctx context.Context, userAuthToken, serviceAuthToken, collectionID, id, edition, version, dimension string) (m dataset.Options, err error)
 }
 
 // OutputQueue - An interface used to queue filter outputs
@@ -34,7 +33,7 @@ type OutputQueue interface {
 
 // PreviewDataset An interface used to generate previews
 type PreviewDataset interface {
-	GetPreview(ctx context.Context, filter *models.Filter, limit int) (*preview.FilterPreview, error)
+	GetPreview(ctx context.Context, filter *models.Filter, limit int) (*models.FilterPreview, error)
 }
 
 // FilterAPI manages importing filters against a dataset
@@ -48,30 +47,42 @@ type FilterAPI struct {
 	downloadServiceURL   string
 	downloadServiceToken string
 	auditor              audit.AuditorService
+	serviceAuthToken     string
 }
 
 // CreateFilterAPI manages all the routes configured to API
-func CreateFilterAPI(host, bindAddr, zebedeeURL string,
+func CreateFilterAPI(ctx context.Context,
+	host, bindAddr, zebedeeURL string,
 	datastore DataStore,
 	outputQueue OutputQueue,
 	errorChan chan error,
 	datasetAPI DatasetAPI,
 	preview PreviewDataset,
 	enablePrivateEndpoints bool,
-	downloadServiceURL, downloadServiceToken string,
-	auditor audit.AuditorService) {
+	downloadServiceURL, downloadServiceToken, serviceAuthToken string,
+	auditor audit.AuditorService,
+	hc *healthcheck.HealthCheck) {
 
 	router := mux.NewRouter()
-	routes(host, router, datastore, outputQueue, datasetAPI, preview, enablePrivateEndpoints, downloadServiceURL, downloadServiceToken, auditor)
-
-	healthcheckHandler := healthcheck.NewMiddleware(healthcheck.Do)
-	middlewareChain := alice.New(healthcheckHandler)
-
-	middlewareChain.Append(collectionID.CheckHeader)
+	routes(host,
+		router,
+		datastore,
+		outputQueue,
+		datasetAPI,
+		preview,
+		enablePrivateEndpoints,
+		downloadServiceURL,
+		downloadServiceToken,
+		serviceAuthToken,
+		auditor)
+
+	healthCheckHandler := newMiddleware(hc.Handler)
+	middlewareChain := alice.New(
+		healthCheckHandler,
+		collectionID.CheckHeader)
 
 	if enablePrivateEndpoints {
-
-		log.Debug("private endpoints are enabled. using identity middleware", nil)
+		log.Event(ctx, "private endpoints are enabled. using identity middleware", log.INFO)
 		identityHandler := identity.Handler(zebedeeURL)
 		middlewareChain = middlewareChain.Append(identityHandler)
 	}
@@ -83,24 +94,28 @@ func CreateFilterAPI(host, bindAddr, zebedeeURL string,
 	httpServer.HandleOSSignals = false
 
 	go func() {
-		log.Debug("Starting api...", nil)
+		log.Event(ctx, "Starting api...", log.INFO)
 		if err := httpServer.ListenAndServe(); err != nil {
-			log.ErrorC("api http server returned error", err, nil)
 			errorChan <- err
 		}
 	}()
 }
 
+// newMiddleware creates a new http.Handler to intercept /health requests.
+func newMiddleware(healthcheckHandler func(http.ResponseWriter, *http.Request)) func(http.Handler) http.Handler {
+	return func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			if req.Method == "GET" && req.URL.Path == "/health" {
+				healthcheckHandler(w, req)
+				return
+			}
+			h.ServeHTTP(w, req)
+		})
+	}
+}
+
 // routes contain all endpoints for API
-func routes(host string,
-	router *mux.Router,
-	dataStore DataStore,
-	outputQueue OutputQueue,
-	datasetAPI DatasetAPI,
-	preview PreviewDataset,
-	enablePrivateEndpoints bool,
-	downloadServiceURL, downloadServiceToken string,
-	auditor audit.AuditorService) *FilterAPI {
+func routes(host string, router *mux.Router, dataStore DataStore, outputQueue OutputQueue, datasetAPI DatasetAPI, preview PreviewDataset, enablePrivateEndpoints bool, downloadServiceURL, downloadServiceToken, serviceAuthToken string, auditor audit.AuditorService) *FilterAPI {
 
 	api := FilterAPI{host: host,
 		dataStore:            dataStore,
@@ -111,6 +126,7 @@ func routes(host string,
 		downloadServiceURL:   downloadServiceURL,
 		downloadServiceToken: downloadServiceToken,
 		auditor:              auditor,
+		serviceAuthToken:     serviceAuthToken,
 	}
 
 	api.router.HandleFunc("/filters", api.postFilterBlueprintHandler).Methods("POST")
@@ -142,6 +158,6 @@ func Close(ctx context.Context) error {
 		return err
 	}
 
-	log.InfoCtx(ctx, "graceful shutdown of http server complete", nil)
+	log.Event(ctx, "graceful shutdown of http server complete", log.INFO)
 	return nil
 }
diff --git a/api/api_test.go b/api/api_test.go
@@ -9,7 +9,6 @@ import (
 	"github.com/ONSdigital/dp-filter-api/api/datastoretest"
 	"github.com/ONSdigital/dp-filter-api/filters"
 	"github.com/ONSdigital/dp-filter-api/models"
-	"github.com/ONSdigital/dp-filter-api/preview"
 	"github.com/ONSdigital/go-ns/audit"
 	"github.com/ONSdigital/go-ns/common"
 	. "github.com/smartystreets/goconvey/convey"
@@ -19,7 +18,8 @@ const (
 	host                   = "http://localhost:80"
 	enablePrivateEndpoints = true
 	downloadServiceURL     = "http://localhost:23600"
-	downloadServiceToken   = "123wut"
+	downloadServiceToken   = "123123"
+	serviceAuthToken       = "321321"
 )
 
 var (
@@ -32,8 +32,8 @@ var (
 )
 
 var previewMock = &datastoretest.PreviewDatasetMock{
-	GetPreviewFunc: func(ctx context.Context, filter *models.Filter, limit int) (*preview.FilterPreview, error) {
-		return &preview.FilterPreview{}, nil
+	GetPreviewFunc: func(ctx context.Context, filter *models.Filter, limit int) (*models.FilterPreview, error) {
+		return &models.FilterPreview{}, nil
 	},
 }
 

diff --git a/api/datastore.go b/api/datastore.go
@@ -2,14 +2,14 @@ package api
 
 import (
 	"github.com/ONSdigital/dp-filter-api/models"
-	"github.com/gedge/mgo/bson"
+	"github.com/globalsign/mgo/bson"
 )
 
 //go:generate moq -out datastoretest/datastore.go -pkg datastoretest . DataStore
 
 // DataStore - A interface used to store filters
 type DataStore interface {
-	AddFilter(host string, filter *models.Filter) (*models.Filter, error)
+	AddFilter(filter *models.Filter) (*models.Filter, error)
 	AddFilterDimension(filterID, name string, options []string, dimensions []models.Dimension, timestamp bson.MongoTimestamp) error
 	AddFilterDimensionOption(filterID, name, option string, timestamp bson.MongoTimestamp) error
 	CreateFilterOutput(filter *models.Filter) error