refs : #19042 - listUnusualIPs () function added

Author: remiges-kanchan

logharbour/clientFn.go

@@ -15,6 +15,7 @@ import (
 	"github.com/elastic/go-elasticsearch/v8/typedapi/some"
 	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
 	"github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder"
+	"github.com/oschwald/geoip2-golang"
 )
 
 const (
@@ -37,6 +38,7 @@ const (
 	ACTIVITY    = "A"
 	DEBUG       = "D"
 	field       = "data.change_data.changes.field"
+	EN = "en"
 )
 
 var (
@@ -85,6 +87,13 @@ type GetSetParam struct {
 	RemoteIP *string      `json:"remoteIP" validate:"omitempty"`
 	Pri      *LogPriority `json:"pri" validate:"omitempty,oneof=1 2 3 4 5 6 7 8"`
 }
+type IPLocation struct {
+	Ipaddress string  `json:"ipaddr" validate:"required"`
+	City      string  `json:"city"`
+	Country   string  `json:"country"`
+	Latitude  float64 `json:"lat"`
+	Longitude float64 `json:"long"`
+}
 
 // GetLogs retrieves an slice of logEntry from Elasticsearch based on the fields provided in logParam.
 func GetLogs(querytoken string, client *elasticsearch.TypedClient, logParam GetLogsParam) ([]LogEntry, int, error) {
@@ -200,7 +209,6 @@ func GetLogs(querytoken string, client *elasticsearch.TypedClient, logParam GetL
 		return nil, 0, fmt.Errorf("Error while searching document in es:%v", err)
 	}
 
-
 	// Unmarshalling hit.source into LogEntry
 	if res != nil {
 		for _, hit := range res.Hits.Hits {
@@ -242,14 +250,11 @@ func GetUnusualIP(queryToken string, client *elasticsearch.TypedClient, unusualP
 
 	localIp, err := GetLocalIPAddress()
 	if err != nil {
-		println("Error:", err)
 		return nil, err
 	}
-	println("Local IP address:", localIp)
 
 	if percentThreshold > 1 {
 		for ip, count := range aggregatedIPs {
-			fmt.Printf("IP: %s, Count: %d\n", ip, count)
 			if count <= int64(percentThreshold) {
 				if ip != localIp {
 					unusualIPs = append(unusualIPs, ip)
@@ -728,3 +733,57 @@ func GetChanges(querytoken string, client *elasticsearch.TypedClient, logParam G
 	}
 	return logEntries, int(res.Hits.Total.Value), nil
 }
+
+// This function gives a list of unusual IPS with it's geographical location details
+func ListUnusualIPs(queryToken string, client *elasticsearch.TypedClient, geoLiteDb *geoip2.Reader, unusualPercent float64, logParam GetUnusualIPParam) ([]IPLocation, error) {
+	unusualIPList := []IPLocation{}
+
+	// calling GetUnusualIP() to get all unsualIPS based on app and ndays
+	unusualIPs, err := GetUnusualIP(queryToken, client, unusualPercent, GetUnusualIPParam{App: logParam.App, NDays: logParam.NDays})
+	if err != nil {
+		return nil, err
+	}
+	// getting IPLocation details for each IP address
+	for _, ip := range unusualIPs {
+		iplocation := getIPLocation(ip, geoLiteDb)
+		unusualIPList = append(unusualIPList, iplocation)
+
+	}
+
+	return unusualIPList, nil
+
+}
+
+// This function is used to retrieves the geographical location details of a given IP address using a GeoLite database.
+func getIPLocation(ipAddress string, geoLiteDb *geoip2.Reader) IPLocation {
+
+	var (
+		IPLocationDetails IPLocation
+		IP                net.IP
+	)
+	// parse IP address
+	IP = net.ParseIP(ipAddress)
+
+	// searching city details based on IP address
+	record, err := geoLiteDb.City(IP)
+	if err != nil {
+		IPLocationDetails = IPLocation{
+			Ipaddress: "0.0.0.0",
+			City:      "",
+			Country:   "",
+			Latitude:  0,
+			Longitude: 0,
+		}
+		return IPLocationDetails
+	}
+
+	IPLocationDetails = IPLocation{
+		Ipaddress: ipAddress,
+		City:      record.City.Names[EN],
+		Country:   record.Country.Names[EN],
+		Latitude:  record.Location.Latitude,
+		Longitude: record.Location.Longitude,
+	}
+
+	return IPLocationDetails
+}

server/config_dev.json

@@ -5,5 +5,6 @@
     "db_user": "elastic",
     "index_name": "logharbour",
     "db_password": "9jjWQryjHca-9flzDcKU",
-    "certificate_fingerprint": "3395adb7832f24e043ea7101b7f821bd97786fc808c335ba439a3681585119fc"
+    "certificate_fingerprint": "3395adb7832f24e043ea7101b7f821bd97786fc808c335ba439a3681585119fc",
+    "geolite_db_path" :"../logharbour/GeoLite2-City.mmdb"
 }

server/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/elastic/elastic-transport-go/v8/elastictransport"
 	"github.com/elastic/go-elasticsearch/v8"
 	"github.com/gin-gonic/gin"
+	"github.com/oschwald/geoip2-golang"
 	"github.com/remiges-tech/alya/config"
 	"github.com/remiges-tech/alya/service"
 	"github.com/remiges-tech/alya/wscutils"
@@ -110,34 +111,51 @@ func main() {
 		// show request query logger
 		Logger: &elastictransport.TextLogger{Output: log.Writer(), EnableRequestBody: true},
 	}
+	//elasticsearch client
 	client, err := elasticsearch.NewTypedClient(dbConfig)
 	if err != nil {
 		log.Fatalf("Failed to create db connection: %v", err)
 		wscutils.NewErrorResponse(502, "error establishing a database connection")
 		return
 	}
 
+	// GeoLite2-City database
+	geoLiteCityDb, err := geoip2.Open(appConfig.GeoLiteDbPath)
+	if err != nil {
+		log.Fatalf("Failed to create GeoLite2-City db connection: %v", err)
+		wscutils.NewErrorResponse(502, "error establishing a GeoLite2-City database connection")
+		return
+	}
+	defer geoLiteCityDb.Close()
+
 	// router
 	r := gin.Default()
 
-	// r.Use(corsMiddleware())
+	//	r.Use(corsMiddleware())
+
+	apiV1Group := r.Group("/api/v1/")
 
 	// services
 	s := service.NewService(r).
 		WithLogHarbour(l).
 		WithDependency("client", client).
 		WithDependency("index", "logharbour")
 
-	apiV1Group := r.Group("/api/v1/")
-
 	s.RegisterRouteWithGroup(apiV1Group, http.MethodPost, "/highprilog", wsc.GetHighprilog)
 	s.RegisterRouteWithGroup(apiV1Group, http.MethodPost, "/activitylog", wsc.ShowActivityLog)
 	s.RegisterRouteWithGroup(apiV1Group, http.MethodPost, "/debuglog", wsc.GetDebugLog)
 	s.RegisterRouteWithGroup(apiV1Group, http.MethodPost, "/datachange", wsc.ShowDataChange)
-	s.RegisterRouteWithGroup(apiV1Group, http.MethodPost, "/unusalip", wsc.GetUnusualIP)
 	s.RegisterRouteWithGroup(apiV1Group, http.MethodPost, "/getset", wsc.GetSet)
 	s.RegisterRouteWithGroup(apiV1Group, http.MethodGet, "/getapps", wsc.GetApps)
 
+	// creating a seprate service for getting list of unusualIPS with geoLiteCityDb dependency
+	unusualIPServ := service.NewService(r).
+		WithLogHarbour(l).
+		WithDependency("client", client).
+		WithDependency("index", "logharbour").WithDependency("geoLiteCityDb", geoLiteCityDb)
+
+	unusualIPServ.RegisterRouteWithGroup(apiV1Group, http.MethodPost, "/getunusualips", wsc.GetUnusualIPs)
+
 	err = r.Run(":" + appConfig.AppServerPort)
 	if err != nil {
 		l.LogActivity("Failed to start server", err)

server/types/types.go

@@ -14,6 +14,7 @@ type AppConfig struct {
 	KeycloakURL            string `json:"keycloak_url"`
 	KeycloakClientID       string `json:"keycloak_client_id"`
 	CertificateFingerprint string `json:"certificate_fingerprint"`
+	GeoLiteDbPath          string  `json:"geolite_db_path"`
 }
 
 

server/wsc/getunusualIPs.go

@@ -0,0 +1,83 @@
+package wsc
+
+import (
+	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/gin-gonic/gin"
+	"github.com/go-playground/validator/v10"
+	"github.com/oschwald/geoip2-golang"
+	"github.com/remiges-tech/alya/service"
+	"github.com/remiges-tech/alya/wscutils"
+	"github.com/remiges-tech/logharbour/logharbour"
+)
+
+//var unusualPercent = 10.0
+
+type GetUnusualIpParam struct {
+	App            string  `json:"app" validate:"required,alpha,lt=15"`
+	Days           int     `json:"days" validate:"required,number,lt=500"`
+	UnusualPercent float64 `json:"unusualPercent"`
+}
+type GetUnusualIPResponse struct {
+	UnusualIPs []logharbour.IPLocation `json:"unusualIPs" validate:"required"`
+}
+
+func GetUnusualIPs(c *gin.Context, s *service.Service) {
+	l := s.LogHarbour
+	l.Debug0().Log("starting execution of GetUnusualIPs()")
+
+	var (
+		req        GetUnusualIpParam
+		unusualIPs []logharbour.IPLocation
+	)
+
+	// binding json request
+	err := wscutils.BindJSON(c, &req)
+	if err != nil {
+		l.Debug0().Error(err).Log("error unmarshalling request payload to struct")
+		return
+	}
+
+	// Validate request
+	validationErrors := wscutils.WscValidate(req, func(err validator.FieldError) []string { return []string{} })
+	if len(validationErrors) > 0 {
+		l.Debug0().LogDebug("standard validation errors", validationErrors)
+		wscutils.SendErrorResponse(c, wscutils.NewResponse(wscutils.ErrorStatus, nil, validationErrors))
+		return
+	}
+
+	es, ok := s.Dependencies["client"].(*elasticsearch.TypedClient)
+	if !ok {
+		wscutils.SendErrorResponse(c, wscutils.NewErrorResponse(100, "ErrCode_DatabaseError"))
+		return
+	}
+
+	geoLiteDb, ok := s.Dependencies["geoLiteCityDb"].(*geoip2.Reader)
+	if !ok {
+		wscutils.SendErrorResponse(c, wscutils.NewErrorResponse(100, "ErrCode_DatabaseError"))
+		return
+	}
+
+	unusualIPs, err = logharbour.ListUnusualIPs("", es, geoLiteDb, req.UnusualPercent, logharbour.GetUnusualIPParam{
+		App:   &req.App,
+		NDays: &req.Days,
+	})
+
+	if err != nil {
+		l.Debug0().Error(err).Log("error in GetUnusualIPs")
+		wscutils.SendErrorResponse(c, wscutils.NewErrorResponse(222, err.Error()))
+		return
+	}
+	if len(unusualIPs) != 0 {
+
+		response := GetUnusualIPResponse{
+			UnusualIPs: unusualIPs,
+		}
+
+		l.Debug0().Log("finished execution of GetUnusualIPs()")
+		wscutils.SendSuccessResponse(c, wscutils.NewSuccessResponse(response))
+		return
+	}
+	l.Debug0().Log("starting execution of GetUnusualIPs()")
+	wscutils.SendSuccessResponse(c, wscutils.NewSuccessResponse("No unusual IP"))
+
+}

server/wsc/test/main_test.go

@@ -147,7 +147,7 @@ func registerRoutes(typedClient *es.TypedClient) (*gin.Engine, error) {
 	s.RegisterRoute(http.MethodPost, "/highprilog", wsc.GetHighprilog)
 	s.RegisterRoute(http.MethodPost, "/activitylog", wsc.ShowActivityLog)
 	s.RegisterRoute(http.MethodPost, "/debuglog", wsc.GetDebugLog)
-	s.RegisterRoute(http.MethodPost, "/getUnusualIP", wsc.GetUnusualIP)
+	s.RegisterRoute(http.MethodPost, "/getUnusualIP", wsc.GetUnusualIPs)
 	s.RegisterRoute(http.MethodPost, "/datachange", wsc.ShowDataChange)
 	s.RegisterRoute(http.MethodGet, "/getapps", wsc.GetApps)
 	s.RegisterRoute(http.MethodPost, "/getset", wsc.GetSet)