Skip to content

Commit a914adb

Browse files
remiges-kanchanremiges-kanchan
committed
refactor : moved ElasticSearchUtils functions to test the data
1 parent 56da679 commit a914adb

File tree

5 files changed

+228
-40
lines changed

5 files changed

+228
-40
lines changed

‎logharbour/clientFn.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -204,11 +204,11 @@ func GetLogs(querytoken string, client *elasticsearch.TypedClient, logParam GetL
204204
// Unmarshalling hit.source into LogEntry
205205
if res != nil {
206206
for _, hit := range res.Hits.Hits {
207-
var logEnter LogEntry
208-
if err := json.Unmarshal([]byte(hit.Source_), &logEnter); err != nil {
207+
var logEntery LogEntry
208+
if err := json.Unmarshal([]byte(hit.Source_), &logEntery); err != nil {
209209
return nil, 0, fmt.Errorf("error while unmarshalling response:%v", err)
210210
}
211-
logEntries = append(logEntries, logEnter)
211+
logEntries = append(logEntries, logEntery)
212212
}
213213
}
214214
return logEntries, int(res.Hits.Total.Value), nil

‎logharbour/test/elastic_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ import (
77
"time"
88

99
"github.com/remiges-tech/logharbour/logharbour"
10-
elasticsearchctl "github.com/remiges-tech/logharbour/server/elasticSearchCtl/elasticSearch"
10+
estestutils "github.com/remiges-tech/logharbour/logharbour/test"
11+
1112
"github.com/stretchr/testify/require"
1213
)
1314

@@ -55,7 +56,7 @@ func TestGetLogs(t *testing.T) {
5556
t.Run(tc.Name, func(t *testing.T) {
5657

5758
if tc.TestJsonFile != "" {
58-
tc.ExpectedLogEntries, err = elasticsearchctl.ReadLogFromFile(tc.TestJsonFile)
59+
tc.ExpectedLogEntries, err = estestutils.ReadLogFromFile(tc.TestJsonFile)
5960
if err != nil {
6061
fmt.Printf("error converting data from log file:%v\n", err)
6162
}

‎logharbour/test/estestutils.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package logharbour
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"log"
8+
"os"
9+
"strconv"
10+
"strings"
11+
12+
"github.com/elastic/go-elasticsearch/v8"
13+
"github.com/elastic/go-elasticsearch/v8/esapi"
14+
"github.com/remiges-tech/logharbour/logharbour"
15+
)
16+
17+
// Function CreateElasticIndex create an Elasticsearch index with the provided index name and index body.
18+
func CreateElasticIndex(es *elasticsearch.Client, indexName string, indexBody string) error {
19+
20+
// Create the index request
21+
req := esapi.IndicesCreateRequest{
22+
Index: indexName,
23+
Body: strings.NewReader(indexBody),
24+
}
25+
26+
// Perform the request
27+
res, err := req.Do(context.Background(), es)
28+
if err != nil {
29+
return fmt.Errorf("error creating the index: %s", err)
30+
}
31+
32+
defer res.Body.Close()
33+
34+
// Print the response status and body
35+
fmt.Println("Response status:", res.Status())
36+
if res.IsError() {
37+
var errorResponse map[string]interface{}
38+
if err := json.NewDecoder(res.Body).Decode(&errorResponse); err != nil {
39+
return fmt.Errorf("error parsing the error response body: %s", err)
40+
}
41+
log.Fatalf("Error creating the index: %s", errorResponse["error"].(map[string]interface{})["reason"])
42+
} else {
43+
44+
fmt.Println("Index created successfully:")
45+
}
46+
return nil
47+
48+
}
49+
50+
// Function InsertLog bulk insert an array of log entries into an Elasticsearch index using the provided Elasticsearch client.
51+
func InsertLog(es *elasticsearch.Client, logs []logharbour.LogEntry, indexName string) error {
52+
53+
for i, log := range logs {
54+
dataJson, err := json.Marshal(log)
55+
if err != nil {
56+
return fmt.Errorf("error while unmarshaling log: %v", err)
57+
}
58+
59+
js := string(dataJson)
60+
61+
req := esapi.IndexRequest{
62+
Index: indexName,
63+
DocumentID: strconv.Itoa(i + 1),
64+
Body: strings.NewReader(js),
65+
Refresh: "true",
66+
}
67+
68+
res, err := req.Do(context.Background(), es)
69+
if err != nil {
70+
return fmt.Errorf("error while adding data in es :%v", err)
71+
}
72+
defer res.Body.Close()
73+
if res.IsError() {
74+
return fmt.Errorf("error indexing document ID=%s", res)
75+
}
76+
}
77+
return nil
78+
}
79+
80+
// Function ReadLogFromFile read log entries from a file, unmarshal the file data, and return slice of LogEntry
81+
func ReadLogFromFile(filepath string) ([]logharbour.LogEntry, error) {
82+
83+
byteValue, err := os.ReadFile(filepath)
84+
if err != nil {
85+
return nil, err
86+
}
87+
88+
var LogEntries []logharbour.LogEntry
89+
90+
err = json.Unmarshal(byteValue, &LogEntries)
91+
if err != nil {
92+
return nil, err
93+
}
94+
return LogEntries, nil
95+
}

‎logharbour/test/main_test.go

Lines changed: 64 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,18 @@ import (
99
"time"
1010

1111
es "github.com/elastic/go-elasticsearch/v8"
12-
elasticsearchctl "github.com/remiges-tech/logharbour/server/elasticSearchCtl/elasticSearch"
12+
estestutils "github.com/remiges-tech/logharbour/logharbour/test"
1313
"github.com/testcontainers/testcontainers-go"
1414
"github.com/testcontainers/testcontainers-go/modules/elasticsearch"
1515
)
1616

1717
var (
1818
indexBody = `{
19-
"settings": {
20-
"number_of_shards": 1,
21-
"number_of_replicas": 0
22-
},
2319
"mappings": {
2420
"properties": {
21+
"id": {
22+
"type": "keyword"
23+
},
2524
"app": {
2625
"type": "keyword"
2726
},
@@ -56,16 +55,70 @@ var (
5655
"type": "integer"
5756
},
5857
"error": {
59-
"type": "keyword"
58+
"type": "text"
6059
},
6160
"remote_ip": {
6261
"type": "ip"
6362
},
6463
"msg": {
65-
"type": "keyword"
64+
"type": "text"
6665
},
6766
"data": {
68-
"type": "text"
67+
"properties": {
68+
"change_data": {
69+
"properties": {
70+
"entity": {
71+
"type": "keyword"
72+
},
73+
"op": {
74+
"type": "keyword"
75+
},
76+
"changes": {
77+
"type": "nested",
78+
"properties": {
79+
"field": {
80+
"type": "keyword"
81+
},
82+
"old_value": {
83+
"type": "text"
84+
},
85+
"new_value": {
86+
"type": "text"
87+
}
88+
}
89+
}
90+
}
91+
},
92+
"activity_data": {
93+
"type": "text"
94+
},
95+
"debug_data": {
96+
"properties": {
97+
"pid": {
98+
"type": "integer"
99+
},
100+
"runtime": {
101+
"type": "keyword"
102+
},
103+
"file": {
104+
"type": "keyword"
105+
},
106+
"line": {
107+
"type": "integer"
108+
},
109+
"func": {
110+
"type": "keyword"
111+
},
112+
"stackTrace": {
113+
"type": "text"
114+
},
115+
"data": {
116+
"type": "object",
117+
"enabled": false
118+
}
119+
}
120+
}
121+
}
69122
}
70123
}
71124
}
@@ -129,16 +182,16 @@ func TestMain(m *testing.M) {
129182

130183
func fillElasticWithData(esClient *es.Client, indexName, indexBody, filepath string) error {
131184

132-
if err := elasticsearchctl.CreateElasticIndex(esClient, indexName, indexBody); err != nil {
185+
if err := estestutils.CreateElasticIndex(esClient, indexName, indexBody); err != nil {
133186
return fmt.Errorf("error while creating elastic search index: %v", err)
134187
}
135188

136-
logEntries, err := elasticsearchctl.ReadLogFromFile(filepath)
189+
logEntries, err := estestutils.ReadLogFromFile(filepath)
137190
if err != nil {
138191
return fmt.Errorf("error converting data from log file:%v", err)
139192
}
140193

141-
if err := elasticsearchctl.InsertLog(esClient, logEntries, indexName); err != nil {
194+
if err := estestutils.InsertLog(esClient, logEntries, indexName); err != nil {
142195
return fmt.Errorf("error while inserting data in elastic search: %v", err.Error())
143196
}
144197

‎server/wsc/test/main_test.go

Lines changed: 63 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,19 @@ import (
1414
"github.com/remiges-tech/alya/service"
1515
"github.com/remiges-tech/alya/wscutils"
1616
"github.com/remiges-tech/logharbour/logharbour"
17-
elasticsearchctl "github.com/remiges-tech/logharbour/server/elasticSearchCtl/elasticSearch"
17+
estestutils "github.com/remiges-tech/logharbour/logharbour/test"
1818
"github.com/remiges-tech/logharbour/server/wsc"
1919
"github.com/testcontainers/testcontainers-go"
2020
"github.com/testcontainers/testcontainers-go/modules/elasticsearch"
2121
)
2222

2323
var (
2424
indexBody = `{
25-
"settings": {
26-
"number_of_shards": 1,
27-
"number_of_replicas": 0
28-
},
2925
"mappings": {
3026
"properties": {
27+
"id": {
28+
"type": "keyword"
29+
},
3130
"app": {
3231
"type": "keyword"
3332
},
@@ -62,35 +61,75 @@ var (
6261
"type": "integer"
6362
},
6463
"error": {
65-
"type": "keyword"
64+
"type": "text"
6665
},
6766
"remote_ip": {
6867
"type": "ip"
6968
},
7069
"msg": {
71-
"type": "keyword"
70+
"type": "text"
7271
},
7372
"data": {
74-
"type": "object"
75-
},
76-
"data.entity": {
77-
"type": "keyword"
78-
},
79-
"data.op": {
80-
"type": "keyword"
81-
},
82-
"data.changes.field": {
83-
"type": "keyword"
84-
},
85-
"data.changes.new_value": {
73+
"properties": {
74+
"change_data": {
75+
"properties": {
76+
"entity": {
77+
"type": "keyword"
78+
},
79+
"op": {
80+
"type": "keyword"
81+
},
82+
"changes": {
83+
"type": "nested",
84+
"properties": {
85+
"field": {
86+
"type": "keyword"
87+
},
88+
"old_value": {
89+
"type": "text"
90+
},
91+
"new_value": {
92+
"type": "text"
93+
}
94+
}
95+
}
96+
}
97+
},
98+
"activity_data": {
8699
"type": "text"
87100
},
88-
"data.changes.old_value": {
89-
"type": "text"
101+
"debug_data": {
102+
"properties": {
103+
"pid": {
104+
"type": "integer"
105+
},
106+
"runtime": {
107+
"type": "keyword"
108+
},
109+
"file": {
110+
"type": "keyword"
111+
},
112+
"line": {
113+
"type": "integer"
114+
},
115+
"func": {
116+
"type": "keyword"
117+
},
118+
"stackTrace": {
119+
"type": "text"
120+
},
121+
"data": {
122+
"type": "object",
123+
"enabled": false
124+
}
90125
}
126+
}
127+
}
128+
}
91129
}
92130
}
93131
}`
132+
94133
typedClient *es.TypedClient
95134
r *gin.Engine
96135
seedDataFilePath = "../test/seed_data.json"
@@ -158,16 +197,16 @@ func TestMain(m *testing.M) {
158197

159198
func fillElasticWithData(esClient *es.Client, indexName, indexBody, filepath string) error {
160199

161-
if err := elasticsearchctl.CreateElasticIndex(esClient, indexName, indexBody); err != nil {
200+
if err := estestutils.CreateElasticIndex(esClient, indexName, indexBody); err != nil {
162201
return fmt.Errorf("error while creating elastic search index: %v", err)
163202
}
164203

165-
logEntries, err := elasticsearchctl.ReadLogFromFile(filepath)
204+
logEntries, err := estestutils.ReadLogFromFile(filepath)
166205
if err != nil {
167206
return fmt.Errorf("error converting data from log file:%v", err)
168207
}
169208

170-
if err := elasticsearchctl.InsertLog(esClient, logEntries, indexName); err != nil {
209+
if err := estestutils.InsertLog(esClient, logEntries, indexName); err != nil {
171210
return fmt.Errorf("error while inserting data in elastic search: %v", err.Error())
172211
}
173212

0 commit comments

Comments
 (0)