Stuff for the first iteration of 2.0

Author: leandrosansilva

.gitlab-ci.yml

@@ -91,7 +91,7 @@ test:
   needs: []
   script:
     - make mocks
-    - go test ./... -race -v | go run github.com/jstemmer/go-junit-report > report.xml
+    - go test ./... -race -v -tags="sqlite_json" | go run github.com/jstemmer/go-junit-report > report.xml
   artifacts:
     reports:
       junit: report.xml
@@ -103,7 +103,7 @@ cover:
   needs: []
   script:
     - go get -u github.com/t-yuki/gocover-cobertura
-    - ./tools/go_test.sh -coverprofile=coverinfo_with_gen.txt
+    - ./tools/go_test.sh -tags sqlite_json -coverprofile=coverinfo_with_gen.txt
     - grep -v '.gen.go' < coverinfo_with_gen.txt > coverinfo.txt
     - go run github.com/t-yuki/gocover-cobertura < coverinfo.txt > cobertura.xml
     - go tool cover -html=coverinfo.txt -o coverage.html

.golangci.yml

@@ -13,6 +13,7 @@ run:
 linters:
   enable-all: true
   disable:
+    - gomoddirectives # TODO: remove this as soon as as https://github.com/imdario/mergo/pull/198 and https://github.com/imdario/mergo/pull/197 are merged
     - promlinter
     - nolintlint
     - thelper

Makefile

@@ -22,7 +22,7 @@ all:
 	$(error Use make (dev|release|static_release) instead)
 
 race:
-	./tools/go_test.sh -race
+	./tools/go_test.sh -race -tags="sqlite_json"
 
 BUILD_DEPENDENCIES = go gcc ragel npm vue
 $(foreach exec,$(BUILD_DEPENDENCIES),\
@@ -37,16 +37,16 @@ pre_build: npminstall translations frontend_root static_www postfix_parser domai
 pre_release: pre_build recommendation_release
 
 dev_bin: dev_pre_build recommendation_dev
-	go build -tags="dev include no_postgres no_mysql no_clickhouse no_mssql" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
+	go build -tags="dev include no_postgres no_mysql no_clickhouse no_mssql sqlite_json" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
 
 dev_headless_bin: dev_headless_pre_build recommendation_dev
-	go build -tags="dev include no_postgres no_mysql no_clickhouse no_mssql" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
+	go build -tags="dev include no_postgres no_mysql no_clickhouse no_mssql sqlite_json" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
 
 release_bin: pre_release
-	go build -tags="release include no_postgres no_mysql no_clickhouse no_mssql" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
+	go build -tags="release include no_postgres no_mysql no_clickhouse no_mssql sqlite_json" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
 
 static_release_bin: pre_release
-	go build -tags="release include no_postgres no_mysql no_clickhouse no_mssql" -o "lightmeter" -ldflags \
+	go build -tags="release include no_postgres no_mysql no_clickhouse no_mssql sqlite_json" -o "lightmeter" -ldflags \
 		"${BUILD_INFO_FLAGS} -linkmode external -extldflags '-static' -s -w" -a -v
 
 static_www:
@@ -66,16 +66,19 @@ recommendation_dev:
 recommendation_release:
 	go generate -tags="release" gitlab.com/lightmeter/controlcenter/recommendation
 
-mocks: postfix_parser dashboard_mock insights_mock detective_mock
+mocks: postfix_parser dashboard_mock insights_mock detective_mock intel_mock
 
 dashboard_mock:
 	go generate -tags="dev" gitlab.com/lightmeter/controlcenter/dashboard
 
 insights_mock:
 	go generate -tags="dev" gitlab.com/lightmeter/controlcenter/insights/core
 
+intel_mock:
+	go generate -tags="dev" gitlab.com/lightmeter/controlcenter/intel/receptor
+
 detective_mock:
-	go generate -tags="dev" gitlab.com/lightmeter/controlcenter/detective
+	go generate -tags="dev sqlite_json" gitlab.com/lightmeter/controlcenter/detective
 
 po2go:
 	go generate -tags="dev" gitlab.com/lightmeter/controlcenter/po

README.md

@@ -200,7 +200,7 @@ Running headless mode requires building ControlCenter for this purpose:
 
 ```
 make devheadless # Build ControlCenter
-./lightmeter -stdin -verbose --listen :8003 # Example command to start ControlCenter quickly (same as running a normal build)
+./lightmeter -stdin -log-level DEBUG --listen :8003 # Example command to start ControlCenter quickly (same as running a normal build)
 ```
 
 ### Authentication
@@ -235,7 +235,22 @@ For detailed information, check [Usage](cli_usage.md).
 (even if compressed with gzip or bzip2) and waiting new log files that happen after such import.
 To use it, start lightmeter with the argument `-watch_dir /path/to/dir`, which is likely to be `/var/log/mail`.
 Lightmeter won't import such logs again if they have already been imported, in case of a process restart.
-- You can run it behind a reverse http proxy such as Apache httpd or nginx, even on a different path. No extra configuration is needed.
+
+### Reverse proxy and securing the user interface
+
+Control Center has a web based user interface, provided over a ***plain*** HTTP server. It does not ship a HTTP server.
+It's up to you to deploy it behind a HTTP reverse proxy, such as NGINX, Apache HTTPD, H2 or Traefik.
+
+Running behind a reverse proxy does not require any extra configuration, even if Control Center user interface is available
+in a path different from `/`, for instance, on `https://example.com/lightmeter/`.
+
+Control Center assumes that it's running behind a reverse proxy to obtain the user's IP (used for rate limiting requests and prevent some sorts of DOS attacks).
+
+In case you are running Control Center in a private network and do not need to run it behind a reverse proxy, using plain HTTP, you should pass
+an extra argument in the command line `-i_know_what_am_doing_not_using_a_reverse_proxy` or set the environment variable
+`LIGHTMETER_I_KNOW_WHAT_I_AM_DOING_NOT_USING_A_REVERSE_PROXY=true`.
+
+###
 
 Currently the following patterns for log files are "watched":
 - mail.log
@@ -253,13 +268,14 @@ For cloud installations of Lightmeter, it may be necessary to set parameters via
 Here are all parameters you can set through environment variables, and their respective command-line equivalents:
 - `LIGHTMETER_WORKSPACE=/path/to/workspace` (`-workspace`)
 - `LIGHTMETER_WATCH_DIR=/var/log` (`-watch_dir`)
-- `LIGHTMETER_VERBOSE=true` (`-verbose`)
+- `LIGHTMETER_LOG_LEVEL=DEBUG` (`-log-level DEBUG`)
 - `LIGHTMETER_LISTEN=localhost:9999` (`-listen`)
 - `LIGHTMETER_LOGS_SOCKET=unix;/path/to/socket.sock` (`-logs_socket`)
 - `LIGHTMETER_LOGS_USE_RSYNC=true` (`-logs_use_rsync`)
 - `LIGHTMETER_LOGS_STARTING_YEAR=2019` (`-log_starting_year`)
 - `LIGHTMETER_LOG_FORMAT=prepend-rfc3339` (`-log_format`)
 - `LIGHTMETER_LOG_FILE_PATTERNS=mail.log:mail.err:mail.warn:zimbra.log:maillog` (`-log_file_patterns`)
+- `LIGHTMETER_I_KNOW_WHAT_I_AM_DOING_NOT_USING_A_REVERSE_PROXY=true` (`-i_know_what_am_doing_not_using_a_reverse_proxy`)
 
 ### Rotated files
 

acceptance_tests/specs/1_1_registration.spec

@@ -7,7 +7,7 @@ Tags: failure
 * Focus on field with placeholder "Name"
 * Type "User Complete Name"
 * Focus on field with placeholder "Email"
-* Type "[email protected]"
+* Type "[email protected]"
 * Focus on field with placeholder "Password"
 * Type "54353%#%#54354353gffgdgdfg"
 * Expect registration to fail
@@ -20,7 +20,7 @@ Tags: success
 * Focus on field with placeholder "Name"
 * Type "User Complete Name"
 * Focus on field with placeholder "Email"
-* Type "[email protected]"
+* Type "[email protected]"
 * Focus on field with placeholder "Password"
 * Type "54353%#%#54354353gffgdgdfg"
 * Select option "direct" from menu "Most of my mail is…"

acceptance_tests/specs/3_login.spec

@@ -5,7 +5,7 @@ Tags: success
 
 * Go to login page
 * Focus on field with placeholder "Email"
-* Type "[email protected]"
+* Type "[email protected]"
 * Focus on field with placeholder "Password"
 * Type "54353%#%#54354353gffgdgdfg"
 * Click on "Login"

acceptance_tests/tests/step_implementation.js

@@ -38,10 +38,18 @@ const headless = process.env.headless_chrome.toLowerCase() === 'true';
 
 var lightmeterProcess = null
 
-var tmpDir = tmp.dirSync()
+var workspaceDir = tmp.dirSync()
 
 beforeSuite(async () => {
-    lightmeterProcess = child_process.execFile('../lightmeter', ['-workspace', tmpDir.name, '-stdin', '-listen', ':8080'])
+    let callback = function(error, stdout, stderr) {
+      if (error) {
+        console.warn(stdout)
+        console.error(stderr)
+        throw error
+      }
+    }
+
+    lightmeterProcess = child_process.execFile('../lightmeter', ['-workspace', workspaceDir.name, '-stdin', '-listen', ':8080'], callback)
 
     return new Promise((r) => setTimeout(r, 2000)).then(async () => {
         await openBrowser({ headless: headless, args: ["--no-sandbox", "--no-first-run"] })

api/connectionstats.go

@@ -14,18 +14,18 @@ import (
 	"time"
 )
 
-type smtpAuthAttemptsHandler struct {
+type authAttemptsHandler struct {
 	accessor *connectionstats.Accessor
 }
 
-// @Summary Information Fetch SMTP authentication attempts
+// @Summary Information Fetch authentication attempts
 // @Param from query string true "Initial date in the format 1999-12-23"
 // @Param to   query string true "Final date in the format 1999-12-23"
 // @Produce json
 // @Success 200 {object} connectionstats.Stats "desc"
 // @Failure 422 {string} string "desc"
-// @Router /api/v0/fetchSmtpAuthAttempts [get]
-func (handler smtpAuthAttemptsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
+// @Router /api/v0/fetchAuthAttempts [get]
+func (handler authAttemptsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
 	interval := httpmiddleware.GetIntervalFromContext(r)
 
 	result, err := handler.accessor.FetchAuthAttempts(r.Context(), interval)
@@ -38,5 +38,5 @@ func (handler smtpAuthAttemptsHandler) ServeHTTP(w http.ResponseWriter, r *http.
 
 func HttpConnectionsDashboard(auth *auth.Authenticator, mux *http.ServeMux, timezone *time.Location, accessor *connectionstats.Accessor) {
 	authenticated := httpmiddleware.WithDefaultStack(auth, httpmiddleware.RequestWithInterval(timezone))
-	mux.Handle("/api/v0/fetchSmtpAuthAttempts", authenticated.WithEndpoint(smtpAuthAttemptsHandler{accessor: accessor}))
+	mux.Handle("/api/v0/fetchAuthAttempts", authenticated.WithEndpoint(authAttemptsHandler{accessor: accessor}))
 }

api/detective.go

@@ -22,7 +22,7 @@ import (
 	"time"
 )
 
-func requireDetectiveAuth(auth *httpauth.Authenticator, settingsReader *metadata.Reader) httpmiddleware.Middleware {
+func requireDetectiveAuth(auth *httpauth.Authenticator, settingsReader metadata.Reader) httpmiddleware.Middleware {
 	return func(h httpmiddleware.CustomHTTPHandler) httpmiddleware.CustomHTTPHandler {
 		return httpmiddleware.CustomHTTPHandler(func(w http.ResponseWriter, r *http.Request) error {
 			/* The detective handler can be accessed if authenticated
@@ -128,8 +128,13 @@ func (h oldestAvailableTimeHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 	return httputil.WriteJson(w, OldestAvailableTimeResponse{Time: &time}, http.StatusOK)
 }
 
-func HttpDetective(auth *auth.Authenticator, mux *http.ServeMux, timezone *time.Location, detective detective.Detective, escalator escalator.Requester, settingsReader *metadata.Reader) {
-	publicIfEnabled := httpmiddleware.New(httpmiddleware.RequestWithTimeout(httpmiddleware.DefaultTimeout), requireDetectiveAuth(auth, settingsReader))
+func HttpDetective(auth *auth.Authenticator, mux *http.ServeMux, timezone *time.Location, detective detective.Detective, escalator escalator.Requester, settingsReader metadata.Reader, isBehindReverseProxy bool) {
+	publicIfEnabled := httpmiddleware.New(
+		httpmiddleware.RequestWithRateLimit(10*time.Minute, 20, isBehindReverseProxy, httpmiddleware.BlockQuery),
+		httpmiddleware.RequestWithTimeout(httpmiddleware.DefaultTimeout),
+		requireDetectiveAuth(auth, settingsReader),
+	)
+
 	mux.Handle("/api/v0/checkMessageDeliveryStatus", publicIfEnabled.WithEndpoint(checkMessageDeliveryHandler{detective}))
 	mux.Handle("/api/v0/escalateMessage", publicIfEnabled.WithEndpoint(detectiveEscalatorHandler{requester: escalator, detective: detective}))
 	mux.Handle("/api/v0/oldestAvailableTimeForMessageDetective", publicIfEnabled.WithEndpoint(oldestAvailableTimeHandler{detective: detective}))

api/detective_auth_test.go

@@ -64,9 +64,9 @@ func buildTestEnv(t *testing.T) (*httptest.Server, *mock_detective.MockDetective
 	settingsWriter := writeRunner.Writer()
 	settingsReader := handler.Reader
 
-	HttpDetective(auth, mux, time.UTC, detective, &fakeEscalateRequester{}, settingsReader)
+	HttpDetective(auth, mux, time.UTC, detective, &fakeEscalateRequester{}, settingsReader, true)
 
-	httpauth.HttpAuthenticator(mux, auth, settingsReader)
+	httpauth.HttpAuthenticator(mux, auth, settingsReader, true)
 
 	s := httptest.NewServer(mux)