fluent.conf.xray

# JFROG XRAY METRICS SOURCE
<source>
  @type jfrog_metrics
  @id metrics_http_jfrt
  tag jfrog.metrics.xray
  execution_interval 5s
  metric_prefix 'jfrog.xray'
  jpd_url "#{ENV['JPD_URL']}"
  username "#{ENV['JPD_ADMIN_USERNAME']}"
  token "#{ENV['JFROG_ADMIN_TOKEN']}"
  common_jpd "#{ENV['COMMON_JPD']}"
  # request_timeout 30s
  # verify_ssl "#{ENV['SPLUNK_VERIFY_SSL']}"
</source>
# SPLUNK XRAY METRICS OUTPUT
<match jfrog.metrics.**>
  @type splunk_hec
  protocol "#{ENV['SPLUNK_COM_PROTOCOL']}"
  data_type metric
  hec_host "#{ENV['SPLUNK_HEC_HOST']}"
  hec_port "#{ENV['SPLUNK_HEC_PORT']}"
  hec_token "#{ENV['SPLUNK_METRICS_HEC_TOKEN']}"
  flush_interval 5s
  source ${tag}
  index jfrog_splunk_metrics
  metric_name_key metric_name
  metric_value_key value
  insecure_ssl "#{ENV['SPLUNK_INSECURE_SSL']}"
  gzip_compression "#{ENV['SPLUNK_COMPRESS_DATA']}"
</match>

# ALL CALLHOME
<source>
  @type exec
  tag callhome
  command "curl --request GET '#{ENV['JPD_URL']}/xray/api/v1/system/version' -H 'Authorization: Bearer #{ENV['JFROG_ADMIN_TOKEN']}'"
  run_interval 24h
  <parse>
    @type json
  </parse>
</source>
<match callhome>
  @type copy
  <store>
    @type relabel
    @label @jfrogcalhome
  </store>
  <store>
    @type relabel
    @label @heapcallhome
  </store>
</match>
<label @heapcallhome>
  <filter callhome>
    @type record_transformer
    renew_record true
    keep_keys 'productId,features'
    enable_ruby true
    <record>
      app_id '251985503'
      identity "#{ENV['JPD_URL']}"
      event 'splunk-xray-fluentd-config'
      properties ${return({"featureId" => "XrayVersion/" + record["xray_version"], "productId" => "OnPremObservability-Splunk/1.0.0"})}
    </record>
  </filter>
  <match callhome>
    @type http
    endpoint "https://heapanalytics.com/api/track"
    open_timeout 5
    content_type application/json
    <format>
      @type json
    </format>
    <buffer>
      flush_interval 5s
    </buffer>
  </match>
</label>
<label @jfrogcalhome>
  <filter callhome>
    @type record_transformer
    renew_record true
    keep_keys 'productId,features'
    enable_ruby true
    <record>
      productId 'OnPremObservability-Splunk/1.0.0'
      features ${return([{"featureId" => "XrayVersion/" + record["xray_version"]}])}
    </record>
  </filter>
  <match callhome>
    @type http
    endpoint "#{ENV['JPD_URL']}/artifactory/api/system/usage"
    open_timeout 5
    content_type application/json
    <auth>
      method basic
      username "#{ENV['JPD_ADMIN_USERNAME']}"
      password "#{ENV['JFROG_ADMIN_TOKEN']}"
    </auth>
    <format>
      @type json
    </format>
    <buffer>
      flush_interval 5s
    </buffer>
  </match>
</label>

# XRAY VIOLATIONS CONFIG
<source>
  @type jfrog_siem
  tag jfrog.xray.siem.vulnerabilities
  jpd_url "#{ENV['JPD_URL']}"
  username "#{ENV['JPD_ADMIN_USERNAME']}"
  token "#{ENV['JFROG_ADMIN_TOKEN']}"
  pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/jfrog_siem.log.pos"
</source>
<filter jfrog.xray.siem.*>
  @type record_transformer
  enable_ruby true
  <record>
    log_source "jfrog.xray.siem.vulnerabilities"
    category ${record["type"]}
    url ${record["violation_details_url"]}
    signature ${record["watch_name"]}
    cvss ${record["cvss_score"]}
    cve ${record["cve"]}
    vendor_product "Xray"
  </record>
</filter>
<filter jfrog.xray.siem.*>
  @type record_modifier
  <record>
    _impacted_artifacts_url_ ${if record.has_key?('impacted_artifacts_url'); record.delete('impacted_artifacts_url') ; end; nil}
    _type_ ${if record.has_key?('type'); record.delete('type') ; end; nil}
    _watch_name_ ${if record.has_key?('watch_name'); record.delete('watch_name') ; end; nil}
    _violation_details_url_ ${if record.has_key?('violation_details_url'); record.delete('violation_details_url') ; end; nil}
    _cvss_score_ ${if record.has_key?('cvss_score'); record.delete('cvss_score') ; end; nil}
    _cvss_version_ ${if record.has_key?('cvss_version'); record.delete('cvss_version') ; end; nil}
  </record>
  remove_keys _impacted_artifacts_url_, _type_, _watch_name_, _violation_details_url_, _cvss_score_, _cvss_version_
</filter>
<match jfrog.xray.siem.**>
  @type splunk_hec
  protocol "#{ENV['SPLUNK_COM_PROTOCOL']}"
  hec_host "#{ENV['SPLUNK_HEC_HOST']}"
  hec_port "#{ENV['SPLUNK_HEC_PORT']}"
  hec_token "#{ENV['SPLUNK_HEC_TOKEN']}"
  index jfrog_splunk
  format json
  sourcetype_key log_source
  use_fluentd_time false
  insecure_ssl "#{ENV['SPLUNK_INSECURE_SSL']}"
  # ssl parameter
  # use_ssl true
  # ca_file /path/to/ca.pem
  <buffer>
    flush_interval 1s
    # frequency of the buffer flush
    flush_thread_count 5
    # The number of threads to flush/write chunks in parallel
    chunk_limit_records 10
    # The max number of events that each chunks can store in it
  </buffer>
</match>

# SOURCE DIRECTIVES
# SERVICE LOGS
<source>
  @type tail
  @id xray_server_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-server-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-server-service.log.pos"
  tag jfrog.xray.server.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id xray_persist_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-persist-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-persist-service.log.pos"
  tag jfrog.xray.persist.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id xray_indexer_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-indexer-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-indexer-service.log.pos"
  tag jfrog.xray.indexer.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id xray_analysis_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-analysis-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-analysis-service.log.pos"
  tag jfrog.xray.analysis.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id xray_router_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log.pos"
  tag jfrog.xray.router.service
  <parse>
    @type none
  </parse>
</source>
# TRAEFIK LOGS
<source>
  @type tail
  @id xray_router_traefik_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log.pos"
  tag jfrog.xray.router.traefik
  <parse>
      @type regexp
      expression ^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.+)$
      time_key log_timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
# REQUEST LOGS
<source>
  @type tail
  @id xray_router_request_tail
  path  "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
  tag jfrog.xray.router.request
  <parse>
    @type json
    time_key time
    time_format %Y-%m-%dT%H:%M:%S%:z
  </parse>
</source>
<source>
  @type tail
  @id xray_request_tail
  path  "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-request.log"
  pos_file  "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-request.log.pos"
  tag jfrog.xray.xray.request
  <parse>
    @type regexp
    expression ^(?<log_timestamp>[^ ]*)\|(?<trace_id>[^ ]*)\|(?<remote_address>[^|]++)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>.*)$
    time_key log_timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>


# FILTER DIRECTIVE
## ALL LOGS
<filter **>
  @type concat
  key message
  multiline_start_regexp /\d{4}-\d{1,2}-\d{1,2}/
  timeout_label @NORMAL
  flush_interval 5
</filter>
<match **>
  @type relabel
  @label @NORMAL
</match>
<label @NORMAL>
## SERVICE LOGS
  <filter jfrog.xray.**.service>
    @type record_transformer
    enable_ruby true
    <record>
      message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')}
    </record>
  </filter>
  <filter jfrog.xray.persist.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
    emit_invalid_record_to_error false
  </filter>
  <filter jfrog.xray.indexer.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
    emit_invalid_record_to_error false
  </filter>
  <filter jfrog.xray.analysis.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  <filter jfrog.xray.router.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  <filter jfrog.xray.server.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  ## ALL LOGS
  <filter jfrog.xray.**>
    @type record_transformer
    <record>
      hostname "#{Socket.gethostname}"
      log_source ${tag}
    </record>
  </filter>

  ####################
  # SPLUNK OUTPUT
  ####################
  # SPLUNK XRAY LOGS OUTPUT
  <match jfrog.xray.**>
    @type splunk_hec
    protocol "#{ENV['SPLUNK_COM_PROTOCOL']}"
    hec_host "#{ENV['SPLUNK_HEC_HOST']}"
    hec_port "#{ENV['SPLUNK_HEC_PORT']}"
    hec_token "#{ENV['SPLUNK_HEC_TOKEN']}"
    index jfrog_splunk
    format json
    sourcetype_key log_source
    use_fluentd_time false
    insecure_ssl "#{ENV['SPLUNK_INSECURE_SSL']}"
    gzip_compression "#{ENV['SPLUNK_COMPRESS_DATA']}"
    # ssl parameter
    # use_ssl true
    # ca_file /path/to/ca.pem
    <buffer>
      flush_interval 1s
      # frequency of the buffer flush
      flush_thread_count 5
      # The number of threads to flush/write chunks in parallel
      chunk_limit_records 10
      # The max number of events that each chunks can store in it
    </buffer>
  </match>

  # TEST OUTPUT
  #<match **>
  #  @type stdout
  #</match>

</label>