Merge branch 'release/v2.23.0'

ONSdigital · Oct 23, 2022 · 1d8bfb3 · 1d8bfb3
2 parents 8d8e086 + e766a20
commit 1d8bfb3
Show file tree

Hide file tree

Showing 8 changed files with 116 additions and 235 deletions.
diff --git a/README.md b/README.md
@@ -46,7 +46,6 @@ Scripts for updating and debugging Kafka can be found [here](https://github.com/
 | MONGODB_IS_SSL               | false                                                        | Switch to use (or not) TLS when connecting to mongodb                                                            |
 | SHUTDOWN_TIMEOUT             | 5s                                                           | The graceful shutdown timeout (`time.Duration` format)                                                           |
 | DATASET_API_URL              | http://localhost:22000                                       | The URL of the Dataset API                                                                                       |
-| DATASET_API_AUTH_TOKEN       | FD0108EA-825D-411C-9B1D-41EF7727F465                         | The token used to access the Dataset API                                                                         |
 | HEALTHCHECK_INTERVAL         | 30s                                                          | Time between self-healthchecks (`time.Duration` format)                                                          |
 | HEALTHCHECK_CRITICAL_TIMEOUT | 90s                                                          | The time taken for the health changes from warning state to critical due to subsystem check failures             |
 | SERVICE_AUTH_TOKEN           | FD0108EA-825D-411C-9B1D-41EF7727F465                         | The token used to identify this service when authenticating                                                      |

diff --git a/api/api.go b/api/api.go
@@ -113,6 +113,10 @@ func Setup(
 	if cfg.EnablePrivateEndpoints {
 		api.Router.Handle("/filter-outputs/{filter_output_id}", assert.FilterOutputType(http.HandlerFunc(api.updateFilterOutputHandler))).Methods("PUT")
 		api.Router.HandleFunc("/filter-outputs/{filter_output_id}/events", api.addEventHandler).Methods("POST")
+	} else if cfg.EnableFilterOutputs {
+		// web journey
+		//identityMiddleware := handlers.Identity(cfg.ZebedeeURL)
+		api.Router.Handle("/filter-outputs/{filter_output_id}", assert.FilterOutputType(http.HandlerFunc(api.updateFilterOutputHandler))).Methods("PUT")
 	}
 
 	return api

diff --git a/api/filter_outputs_test.go b/api/filter_outputs_test.go
@@ -29,6 +29,27 @@ const (
 	filterID3 = "123"
 )
 
+/*
+	func TestPutFilterOutputMiddleware(t *testing.T) {
+		t.Parallel()
+		filterFlexAPIMock := &apimock.FilterFlexAPIMock{}
+		cfg := cfg()
+		cfg.EnableFilterOutputs = true
+		cfg.EnablePrivateEndpoints = false
+
+		Convey("Put filter output errors with wrong token", t, func() {
+			r, err := http.NewRequest("PUT", "http://localhost:22100/filter-outputs/12345678", nil)
+			So(err, ShouldBeNil)
+
+			w := httptest.NewRecorder()
+			filterApi := api.Setup(cfg, mux.NewRouter(), &mock.DataStore{}, &mock.FilterJob{}, &mock.DatasetAPI{}, filterFlexAPIMock)
+			filterApi.Router.ServeHTTP(w, r)
+			So(w.Code, ShouldEqual, http.StatusUnauthorized)
+
+		})
+
+}
+*/
 func TestSuccessfulGetFilterOutput(t *testing.T) {
 	t.Parallel()
 

diff --git a/ci/build.yml b/ci/build.yml
@@ -6,7 +6,7 @@ image_resource:
   type: docker-image
   source:
     repository: golang
-    tag: 1.18.3
+    tag: 1.19.2
 
 inputs:
   - name: dp-filter-api

diff --git a/ci/unit.yml b/ci/unit.yml
@@ -6,7 +6,7 @@ image_resource:
   type: docker-image
   source:
     repository: golang
-    tag: 1.18.3
+    tag: 1.19.2
 
 inputs:
   - name: dp-filter-api

diff --git a/config/config.go b/config/config.go
@@ -1,8 +1,8 @@
 package config
 
 import (
-	"time"
 	"fmt"
+	"time"
 
 	mongodriver "github.com/ONSdigital/dp-mongodb/v3/mongodb"
 
@@ -24,7 +24,6 @@ type Config struct {
 	KafkaSecSkipVerify         bool          `envconfig:"KAFKA_SEC_SKIP_VERIFY"`
 	ShutdownTimeout            time.Duration `envconfig:"SHUTDOWN_TIMEOUT"`
 	DatasetAPIURL              string        `envconfig:"DATASET_API_URL"`
-	DatasetAPIAuthToken        string        `envconfig:"DATASET_API_AUTH_TOKEN"           json:"-"`
 	HealthCheckInterval        time.Duration `envconfig:"HEALTHCHECK_INTERVAL"`
 	HealthCheckCriticalTimeout time.Duration `envconfig:"HEALTHCHECK_CRITICAL_TIMEOUT"`
 	ServiceAuthToken           string        `envconfig:"SERVICE_AUTH_TOKEN"               json:"-"`
@@ -38,6 +37,7 @@ type Config struct {
 	DefaultMaxLimit            int           `envconfig:"DEFAULT_MAXIMUM_LIMIT"`
 	AssertDatasetType          bool          `envconfig:"ASSERT_DATASET_TYPE"`
 	FilterFlexAPIURL           string        `envconfig:"FILTER_FLEX_API_URL"`
+	EnableFilterOutputs        bool          `envconfig:"ENABLE_FILTER_OUTPUTS_CHECK"`
 	MongoConfig
 }
 
@@ -70,7 +70,6 @@ func Get() (*Config, error) {
 		KafkaMaxBytes:              2000000,
 		ShutdownTimeout:            5 * time.Second,
 		DatasetAPIURL:              "http://localhost:22000",
-		DatasetAPIAuthToken:        "FD0108EA-825D-411C-9B1D-41EF7727F465",
 		HealthCheckInterval:        30 * time.Second,
 		HealthCheckCriticalTimeout: 90 * time.Second,
 		MaxRequestOptions:          1000, // Maximum number of options acceptable in an incoming Patch request. Compromise between one option per call (inefficient) and an order of 100k options per call, for census data (memory and computationally expensive)
@@ -80,6 +79,7 @@ func Get() (*Config, error) {
 		ServiceAuthToken:           "FD0108EA-825D-411C-9B1D-41EF7727F465",
 		ZebedeeURL:                 "http://localhost:8082",
 		EnablePrivateEndpoints:     true,
+		EnableFilterOutputs:        false,
 		DownloadServiceURL:         "http://localhost:23600",
 		DownloadServiceSecretKey:   "QB0108EZ-825D-412C-9B1D-41EF7747F462",
 		AssertDatasetType:          false,

diff --git a/go.mod b/go.mod
@@ -1,18 +1,21 @@
 module github.com/ONSdigital/dp-filter-api
 
-go 1.18
+go 1.19
 
 // to avoid 'sonatype-2021-4899' non-CVE Vulnerability
 exclude github.com/gorilla/sessions v1.2.1
 
+//to avoide  [CVE-2022-29153] CWE-918: Server-Side Request Forgery (SSRF)
+exclude github.com/hashicorp/consul/api v1.1.0
+
 require (
 	github.com/ONSdigital/dp-api-clients-go v1.43.0
-	github.com/ONSdigital/dp-api-clients-go/v2 v2.145.0
+	github.com/ONSdigital/dp-api-clients-go/v2 v2.186.0
 	github.com/ONSdigital/dp-healthcheck v1.3.0
-	github.com/ONSdigital/dp-kafka/v2 v2.5.0
+	github.com/ONSdigital/dp-kafka/v2 v2.7.3
 	github.com/ONSdigital/dp-mongodb-in-memory v1.3.1
-	github.com/ONSdigital/dp-mongodb/v3 v3.0.2
-	github.com/ONSdigital/dp-net v1.4.1
+	github.com/ONSdigital/dp-mongodb/v3 v3.3.0
+	github.com/ONSdigital/dp-net v1.5.0
 	github.com/ONSdigital/dp-net/v2 v2.4.0
 	github.com/ONSdigital/go-ns v0.0.0-20210916104633-ac1c1c52327e
 	github.com/ONSdigital/log.go/v2 v2.2.0
@@ -22,47 +25,49 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b
 	github.com/smartystreets/goconvey v1.7.2
-	go.mongodb.org/mongo-driver v1.9.1
+	go.mongodb.org/mongo-driver v1.10.3
 )
 
 require (
-	github.com/Shopify/sarama v1.34.1 // indirect
-	github.com/aws/aws-sdk-go v1.44.44 // indirect
+	github.com/Shopify/sarama v1.37.2 // indirect
+	github.com/aws/aws-sdk-go v1.44.115 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/eapache/go-resiliency v1.3.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/go-avro/avro v0.0.0-20171219232920-444163702c11 // indirect
-	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20220104163920-15ed2e8cf2bd // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
-	github.com/jcmturner/gofork v1.0.0 // indirect
-	github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect
+	github.com/jcmturner/gofork v1.7.6 // indirect
+	github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
-	github.com/klauspost/compress v1.15.7 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/montanaflynn/stats v0.6.6 // indirect
+	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
-	github.com/smartystreets/assertions v1.2.1 // indirect
-	github.com/spf13/afero v1.8.2 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/smartystreets/assertions v1.13.0 // indirect
+	github.com/spf13/afero v1.9.2 // indirect
 	github.com/square/mongo-lock v0.0.0-20220601164918-701ecf357cd7 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.1 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect
-	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
-	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
-	golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
+	golang.org/x/net v0.0.0-20220927171203-f486391704dc // indirect
+	golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect
+	golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect
+	golang.org/x/text v0.3.8 // indirect
 )