Skip to content

Commit 3b08f32

Browse files
stellarsquallstellarsquall
committed
added kibaba + elasticsearch support via filebeat
1 parent 04f51dd commit 3b08f32

File tree

14 files changed

+278
-109
lines changed

14 files changed

+278
-109
lines changed

‎.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
.DS_Store
2+
terraform/.DS_Store
3+
terraform/.terraform/*
4+
bins/
5+
*.log

‎compose/auth-sink ‎auditlogs/.keep

File renamed without changes.

‎compose/.env

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Password for the 'elastic' user (at least 6 characters)
2+
export ELASTIC_PASSWORD=elastic
3+
# Password for the 'kibana_system' user (at least 6 characters)
4+
export KIBANA_PASSWORD=kibana
5+
# Version of Elastic products
6+
export STACK_VERSION=8.1.2
7+
# Set the cluster name
8+
export CLUSTER_NAME=boundary-docker-cluster
9+
# Set to 'basic' or 'trial' to automatically start the 30-day trial
10+
export LICENSE=basic
11+
# Port to expose Elasticsearch HTTP API to the host
12+
export ES_PORT=127.0.0.1:19200
13+
# Port to expose Kibana to the host
14+
export KIBANA_PORT=5601
15+
# Increase or decrease based on the available host memory (in bytes)
16+
export MEM_LIMIT=1073741824

‎compose/all-events

-57
This file was deleted.

‎compose/controller.hcl

+36-25
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ controller {
1010
}
1111

1212
listener "tcp" {
13-
address = "boundary"
13+
address = "0.0.0.0:9200"
1414
purpose = "api"
1515
tls_disable = true
1616
cors_enabled = true
@@ -45,36 +45,47 @@ kms "aead" {
4545
}
4646

4747
events {
48-
audit_enabled = false
49-
observation_enabled = true
50-
sysevents_enabled = true
48+
audit_enabled = true
49+
observations_enabled = true
50+
sysevents_enabled = true
51+
5152
sink "stderr" {
52-
name = "all-events"
53+
name = "all-events"
5354
description = "All events sent to stderr"
5455
event_types = ["*"]
55-
format = "cloudevents-json"
56+
format = "cloudevents-json"
5657
}
58+
5759
sink {
58-
name = "all-events"
59-
description = "All events sent to file"
60-
event_types = ["*"]
61-
format = "cloudevents-json"
60+
name = "audit-sink"
61+
description = "Audit sent to a file"
62+
event_types = ["audit"]
63+
format = "cloudevents-json"
64+
6265
file {
63-
path = "/tmp/"
64-
file_name = "all-events"
66+
path = "/logs"
67+
file_name = "audit.log"
6568
}
66-
}
67-
sink {
68-
name = "auth-sink"
69-
description = "Authentications sent to a file"
70-
event_types = ["observation"]
71-
format = "cloudevents-json"
72-
allow_filters = [
73-
"\"/Data/request_info/Path\" contains \":authenticate\""
74-
]
75-
file {
76-
path = "/tmp/"
77-
file_name = "auth-sink"
69+
70+
audit_config {
71+
audit_filter_overrides {
72+
secret = "encrypt"
73+
sensitive = "hmac-sha256"
74+
}
7875
}
7976
}
80-
}
77+
78+
// sink {
79+
// name = "auth-sink"
80+
// description = "Authentications sent to a file"
81+
// event_types = ["observation"]
82+
// format = "cloudevents-json"
83+
// allow_filters = [
84+
// "\"/Data/request_info/Path\" contains \":authenticate\""
85+
// ]
86+
// file {
87+
// path = "./"
88+
// file_name = "auth-sink.log"
89+
// }
90+
// }
91+
}

‎compose/docker-compose.yml

+157-5
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,13 @@ services:
1818
interval: 3s
1919
timeout: 10s
2020
retries: 5
21+
volumes:
22+
- type: bind
23+
source: ../postgres
24+
target: /etc/postgresql/
2125

2226
db-init:
23-
image: hashicorp/boundary:0.6.2
27+
image: hashicorp/boundary:latest-89196f2ce
2428
command: ["database", "init", "-config", "/boundary/controller.hcl"]
2529
volumes:
2630
- "${PWD}/:/boundary/"
@@ -31,35 +35,183 @@ services:
3135
condition: service_healthy
3236

3337
controller:
34-
image: hashicorp/boundary:0.6.2
38+
image: hashicorp/boundary:latest-89196f2ce
39+
cap_add:
40+
- IPC_LOCK
3541
# command: ["server", "-config", "/boundary/controller.hcl"]
3642
entrypoint: sh -c "sleep 3 && exec boundary server -config /boundary/controller.hcl"
3743
volumes:
3844
- "${PWD}/:/boundary/"
45+
- "../auditlogs/:/logs/"
3946
hostname: boundary
4047
ports:
4148
- "9200:9200"
4249
- "9201:9201"
50+
- "9202:9202"
51+
- "9203:9203"
4352
environment:
4453
- BOUNDARY_PG_URL=postgresql://postgres:postgres@db/boundary?sslmode=disable
54+
- SKIP_CHOWN=true
4555
depends_on:
4656
- db-init
4757
networks:
4858
- default
4959
- worker
60+
healthcheck:
61+
test: ["CMD", "curl", "-f", "http://boundary:9200"]
62+
interval: 3s
63+
timeout: 5s
64+
retries: 5
5065

5166
worker:
52-
image: hashicorp/boundary:0.6.2
67+
image: hashicorp/boundary:latest-89196f2ce
5368
command: ["server", "-config", "/boundary/worker.hcl"]
5469
volumes:
5570
- "${PWD}/:/boundary/"
71+
- "../auditlogs/:/logs/"
5672
hostname: worker
5773
ports:
58-
- "9202:9202"
74+
- "9204:9202"
5975
environment:
6076
- HOSTNAME=worker
6177
depends_on:
6278
- controller
6379
networks:
6480
- default
65-
- worker
81+
- worker
82+
83+
# Filebeat, Elastic, Kibana for visualizing audit events
84+
# Based on: https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-compose-file
85+
setup-elastic:
86+
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
87+
volumes:
88+
- certs:/usr/share/elasticsearch/config/certs
89+
user: "0"
90+
command: >
91+
bash -c '
92+
if [ x${ELASTIC_PASSWORD} == x ]; then
93+
echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
94+
exit 1;
95+
elif [ x${KIBANA_PASSWORD} == x ]; then
96+
echo "Set the KIBANA_PASSWORD environment variable in the .env file";
97+
exit 1;
98+
fi;
99+
if [ ! -f certs/ca.zip ]; then
100+
echo "Creating CA";
101+
bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
102+
unzip config/certs/ca.zip -d config/certs;
103+
fi;
104+
if [ ! -f certs/certs.zip ]; then
105+
echo "Creating certs";
106+
echo -ne \
107+
"instances:\n"\
108+
" - name: elasticsearch\n"\
109+
" dns:\n"\
110+
" - elasticsearch\n"\
111+
" - localhost\n"\
112+
" ip:\n"\
113+
" - 127.0.0.1\n"\
114+
> config/certs/instances.yml;
115+
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
116+
unzip config/certs/certs.zip -d config/certs;
117+
fi;
118+
echo "Setting file permissions"
119+
chown -R root:root config/certs;
120+
find . -type d -exec chmod 750 \{\} \;;
121+
find . -type f -exec chmod 640 \{\} \;;
122+
echo "Waiting for Elasticsearch availability";
123+
until curl -s --cacert config/certs/ca/ca.crt https://elasticsearch:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
124+
echo "Setting kibana_system password";
125+
until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
126+
echo "All done!";
127+
'
128+
healthcheck:
129+
test: ["CMD-SHELL", "[ -f config/certs/elasticsearch/elasticsearch.crt ]"]
130+
interval: 1s
131+
timeout: 5s
132+
retries: 120
133+
134+
elasticsearch:
135+
depends_on:
136+
setup-elastic:
137+
condition: service_healthy
138+
image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
139+
volumes:
140+
- certs:/usr/share/elasticsearch/config/certs
141+
- esdata01:/usr/share/elasticsearch/data
142+
ports:
143+
- ${ES_PORT}:9200
144+
environment:
145+
- node.name=elasticsearch
146+
- cluster.name=${CLUSTER_NAME}
147+
- cluster.initial_master_nodes=elasticsearch
148+
- ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
149+
- bootstrap.memory_lock=true
150+
- xpack.security.enabled=true
151+
- xpack.security.http.ssl.enabled=true
152+
- xpack.security.http.ssl.key=certs/elasticsearch/elasticsearch.key
153+
- xpack.security.http.ssl.certificate=certs/elasticsearch/elasticsearch.crt
154+
- xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
155+
- xpack.security.http.ssl.verification_mode=certificate
156+
- xpack.security.transport.ssl.enabled=true
157+
- xpack.security.transport.ssl.key=certs/elasticsearch/elasticsearch.key
158+
- xpack.security.transport.ssl.certificate=certs/elasticsearch/elasticsearch.crt
159+
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
160+
- xpack.security.transport.ssl.verification_mode=certificate
161+
- xpack.license.self_generated.type=${LICENSE}
162+
mem_limit: ${MEM_LIMIT}
163+
ulimits:
164+
memlock:
165+
soft: -1
166+
hard: -1
167+
healthcheck:
168+
test:
169+
[
170+
"CMD-SHELL",
171+
"curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
172+
]
173+
interval: 10s
174+
timeout: 10s
175+
retries: 120
176+
177+
kibana:
178+
depends_on:
179+
elasticsearch:
180+
condition: service_healthy
181+
image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
182+
volumes:
183+
- certs:/usr/share/kibana/config/certs
184+
- kibanadata:/usr/share/kibana/data
185+
ports:
186+
- ${KIBANA_PORT}:5601
187+
environment:
188+
- SERVERNAME=kibana
189+
- ELASTICSEARCH_HOSTS=https://elasticsearch:9200
190+
- ELASTICSEARCH_USERNAME=kibana_system
191+
- ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
192+
- ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
193+
mem_limit: ${MEM_LIMIT}
194+
healthcheck:
195+
test:
196+
[
197+
"CMD-SHELL",
198+
"curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
199+
]
200+
interval: 10s
201+
timeout: 10s
202+
retries: 120
203+
204+
filebeat:
205+
image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
206+
volumes:
207+
- "../filebeat.docker.yml:/usr/share/filebeat/filebeat.yml"
208+
- "../auditlogs/:/source"
209+
- certs:/certs
210+
211+
volumes:
212+
certs:
213+
driver: local
214+
esdata01:
215+
driver: local
216+
kibanadata:
217+
driver: local

‎compose/worker-sink

Whitespace-only changes.

‎compose/worker.hcl

-5
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,6 @@ worker {
1414
address = "worker"
1515
public_addr = "localhost:9202"
1616
controllers = ["boundary"]
17-
// tags {
18-
// region = ["us-east-1"],
19-
// // type = ["prod"]
20-
// type = ["prod", "database", "postgres", "mysql"]
21-
// }
2217
}
2318

2419
kms "aead" {

‎deploy

+21
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,26 @@
11
#!/bin/bash
22

3+
# fix permissions on local dirs
4+
chmod 777 ./auditlogs
5+
chmod 666 ./auditlogs/audit.log
6+
7+
# # Password for the 'elastic' user (at least 6 characters)
8+
# export ELASTIC_PASSWORD=elastic
9+
# # Password for the 'kibana_system' user (at least 6 characters)
10+
# export KIBANA_PASSWORD=kibana
11+
# # Version of Elastic products
12+
# export STACK_VERSION=8.1.2
13+
# # Set the cluster name
14+
# export CLUSTER_NAME=boundary-docker-cluster
15+
# # Set to 'basic' or 'trial' to automatically start the 30-day trial
16+
# export LICENSE=basic
17+
# # Port to expose Elasticsearch HTTP API to the host
18+
# export ES_PORT=127.0.0.1:19200
19+
# # Port to expose Kibana to the host
20+
# export KIBANA_PORT=5601
21+
# # Increase or decrease based on the available host memory (in bytes)
22+
# export MEM_LIMIT=1073741824
23+
324
function cleanup() {
425
pushd compose
526
docker compose -p boundary rm -fs

‎filebeat.docker.yml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
filebeat.inputs:
2+
- type: log
3+
paths:
4+
- /source/*.log
5+
json.add_error_key: true
6+
7+
output.elasticsearch:
8+
hosts: ["https://elasticsearch:9200"]
9+
ssl.certificate_authorities:
10+
- /certs/ca/ca.crt
11+
username: "elastic"
12+
password: "elastic"

‎postgres/postgresql.conf

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
listen_addresses = '*'
2+
max_connections = 100
3+
work_mem = 50000kB
4+
shared_buffers = 64MB
5+
log_destination = 'stderr'
6+
deadlock_timeout = 100
7+
log_min_messages = info
8+
log_min_error_statement = error
9+
log_error_verbosity = verbose
10+
log_lock_waits = true
11+
log_statement = all
12+
log_duration = on
13+
log_min_duration_statement = 0
14+
log_line_prefix = '%m [%p] (%x) %i '

0 commit comments

Comments
 (0)