feat: Add configuration and install plugins

4lie · Sep 3, 2020 · 8fa7007 · 8fa7007
1 parent 73b29e5
commit 8fa7007
Show file tree

Hide file tree

Showing 9 changed files with 254 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -15,6 +15,82 @@ Fluentd is an open source data collector for unified logging layer.
 ## Role Variables
 
 ``` yaml
+fluentd_sources_conf: []
+# fluentd_sources_conf:
+#   - "type": forward
+#     "params":
+#       "port": 24224
+#       "bind": 0.0.0.0
+#   - "type": dummy
+#     "params":
+#       "dummy": "{\"hello\":\"world\"}"
+#       "tag": testi
+#   - "type": exec
+#     "params":
+#       "command": cmd arg arg
+#       "parse":
+#         "keys": k1 k2 k3
+#       "extract":
+#         "tag_key": k1
+#         "time_key": k2
+#         "time_format": "%Y-%m-%d %H:%M:%S"
+#       "run_interval": 10s
+
+fluentd_outputs_conf: []
+# fluentd_outputs_conf:
+#   - "type": file
+#     "tag": "app.*"
+#     "params":
+#       "path": /tmp/myapp_log
+#       "compress": gzip
+#       "buffer":
+#         "timekey": 1d
+#         "timekey_use_utc": true
+#         "timekey_wait": 10m
+#   - "type": copy
+#     "tag": "*"
+#     "params":
+#       "store":
+#         - "type": file
+#           "path": /tmp/myapp
+#           "compress": gzip
+#           "format":
+#             "localtime": false
+#           "buffer":
+#             "timekey_wait": 10m
+#             "timekey": 86400
+#             "timekey_use_utc": true
+#             "path": /tmp/myapp
+#           "inject":
+#           "time_format": "%Y-%m-%d %H:%M:%S"
+#           "localtime": false
+#         - "type": elasticsearch
+#           "host": fluentd
+#           "port": 9200
+#           "index_name": fluentd
+#           "type_name": fluentd
+
+fluentd_filters_conf: []
+# fluentd_filters_conf:
+#   - "type": grep
+#     "tag": foo.bar
+#     "params":
+#       "regexp":
+#         - "key": message
+#           "pattern": /cool/
+#         - "key": hostname
+#           "pattern": /^web\d+\.example\.com$/
+#       "exclude":
+#         "key": message
+#         "pattern": uncool
+
+# Paths to custom configuration templates
+fluentd_custom_conf: []
+
+# Fluentd plugins that you want to install
+fluentd_plugins: []
+  # - { url: "https://raw.githubusercontent.com/emsearcy/fluent-plugin-gelf/master/lib/fluent/plugin/out_gelf.rb" }
+  # - { name: "fluent-plugin-secure-forward", 'version': 0.2.3 }
 ```
 
 ## Example Playbook

diff --git a/defaults/main.yaml b/defaults/main.yaml
@@ -0,0 +1,76 @@
+fluentd_sources_conf: []
+# fluentd_sources_conf:
+#   - "type": forward
+#     "params":
+#       "port": 24224
+#       "bind": 0.0.0.0
+#   - "type": dummy
+#     "params":
+#       "dummy": "{\"hello\":\"world\"}"
+#       "tag": testi
+#   - "type": exec
+#     "params":
+#       "command": cmd arg arg
+#       "parse":
+#         "keys": k1 k2 k3
+#       "extract":
+#         "tag_key": k1
+#         "time_key": k2
+#         "time_format": "%Y-%m-%d %H:%M:%S"
+#       "run_interval": 10s
+
+fluentd_outputs_conf: []
+# fluentd_outputs_conf:
+#   - "type": file
+#     "tag": "app.*"
+#     "params":
+#       "path": /tmp/myapp_log
+#       "compress": gzip
+#       "buffer":
+#         "timekey": 1d
+#         "timekey_use_utc": true
+#         "timekey_wait": 10m
+#   - "type": copy
+#     "tag": "*"
+#     "params":
+#       "store":
+#         - "type": file
+#           "path": /tmp/myapp
+#           "compress": gzip
+#           "format":
+#             "localtime": false
+#           "buffer":
+#             "timekey_wait": 10m
+#             "timekey": 86400
+#             "timekey_use_utc": true
+#             "path": /tmp/myapp
+#           "inject":
+#           "time_format": "%Y-%m-%d %H:%M:%S"
+#           "localtime": false
+#         - "type": elasticsearch
+#           "host": fluentd
+#           "port": 9200
+#           "index_name": fluentd
+#           "type_name": fluentd
+
+fluentd_filters_conf: []
+# fluentd_filters_conf:
+#   - "type": grep
+#     "tag": foo.bar
+#     "params":
+#       "regexp":
+#         - "key": message
+#           "pattern": /cool/
+#         - "key": hostname
+#           "pattern": /^web\d+\.example\.com$/
+#       "exclude":
+#         "key": message
+#         "pattern": uncool
+
+# Paths to custom configuration templates
+fluentd_custom_conf: []
+
+# Fluentd plugins that you want to install
+fluentd_plugins: []
+  # - { url: "https://raw.githubusercontent.com/emsearcy/fluent-plugin-gelf/master/lib/fluent/plugin/out_gelf.rb" }
+  # - { name: "fluent-plugin-secure-forward", 'version': 0.2.3 }
diff --git a/handlers/main.yaml b/handlers/main.yaml
@@ -1,4 +1,5 @@
-- name: Restart td-agent
+---
+- name: restart td-agent
   service:
     name: td-agent
     state: restarted
diff --git a/tasks/main.yaml b/tasks/main.yaml
@@ -23,4 +23,59 @@
   when: ansible_distribution_major_version == '20'
 
 - name: Execute the fluentd installer
-  command: "/tmp/td-agent.sh"
+  shell: /tmp/td-agent.sh
+
+- name: Remove the fluentd installer
+  file: path=/tmp/td-agent.sh state=absent
+
+- name: Install fluentd plugins via name
+  gem:
+    name: "{{ item.name }}"
+    executable: /opt/td-agent/bin/fluent-gem
+    state: present
+    user_install: false
+    version: "{{ item.version | default(omit) }}"
+  with_items: "{{ fluentd_plugins | selectattr('url', 'undefined') | list }}"
+  notify: restart td-agent
+
+- name: Install fluentd plugins via url
+  get_url:
+    url: "{{ item }}"
+    dest: /etc/td-agent/plugin
+  with_items: "{{ fluentd_plugins | selectattr('name', 'undefined') | map(attribute='url') | list }}"
+  notify: restart td-agent
+
+- name: Create fluentd configs directory
+  file:
+    path: /etc/td-agent/conf.d
+    state: directory
+
+- name: Create fluentd main configuration file
+  template:
+    src: td-agent.conf.j2
+    dest: /etc/td-agent/td-agent.conf
+  notify: restart td-agent
+
+- name: Create fluentd additional configuration files
+  template:
+    src: "{{ item }}.conf.j2"
+    dest: "/etc/td-agent/conf.d/{{ item }}.conf"
+  with_items:
+    - source
+    - filter
+    - match
+    - plugin
+  notify: restart td-agent
+
+- name: Create fluentd custom configuration
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/td-agent/conf.d/{{ item.split('/')[-1] }}"
+  with_items: "{{ fluentd_custom_conf }}"
+  notify: restart td-agent
+
+- name: Ensure fluentd (td-agent) service is enabled
+  service:
+    name: td-agent
+    enabled: true
+    use: service
diff --git a/templates/filter.conf.j2 b/templates/filter.conf.j2
@@ -0,0 +1,12 @@
+# {{ ansible_managed }}
+
+{% for filter in fluentd_filters_conf %}
+<filter {{ filter.tag }}>
+  @type {{ filter.type }}
+{% for key, value in filter.items()|sort(attribute='0')  %}
+{% if key == "params" %}
+  {{ value |fluentd_params_to_text }}
+{% endif %}
+{% endfor %}
+</filter>
+{% endfor %}
diff --git a/templates/match.conf.j2 b/templates/match.conf.j2
@@ -0,0 +1,12 @@
+# {{ ansible_managed }}
+
+{% for output in fluentd_outputs_conf %}
+<match {{ output.tag }}>
+  @type {{ output.type }}
+{% for key, value in output.items()|sort(attribute='0')  %}
+{% if key == "params" %}
+  {{ value |fluentd_params_to_text }}
+{% endif %}
+{% endfor %}
+</match>
+{% endfor %}
diff --git a/templates/plugin.conf.j2 b/templates/plugin.conf.j2
@@ -0,0 +1 @@
+# {{ ansible_managed }}
diff --git a/templates/source.conf.j2 b/templates/source.conf.j2
@@ -0,0 +1,12 @@
+# {{ ansible_managed }}
+
+{% for source in fluentd_sources_conf %}
+<source>
+  @type {{ source.type }}
+{% for key, value in source.items()|sort(attribute='0')  %}
+{% if key != "type" %}
+  {{ value |fluentd_params_to_text }}
+{% endif %}
+{% endfor %}
+</source>
+{% endfor %}
diff --git a/templates/td-agent.conf.j2 b/templates/td-agent.conf.j2
@@ -0,0 +1,7 @@
+# {{ ansible_managed }}
+
+<system>
+  workers {{ ansible_processor_vcpus }}
+</system>
+
+@include /etc/td-agent/conf.d/*.conf