fluent.conf.xray

# JFROG XRAY METRICS SOURCE
<source>
  @type jfrog_metrics
  @id metrics_http_jfrt
  tag jfrog.metrics.xray
  interval 5s
  metric_prefix 'jfrog.xray'
  jpd_url "#{ENV['JPD_URL']}"
  username "#{ENV['JPD_ADMIN_USERNAME']}"
  token "#{ENV['JPD_ADMIN_TOKEN']}"
  common_jpd "#{ENV['COMMON_JPD']}"
  target_platform "NEWRELIC"
</source>
# NEWRELIC XRAY METRICS OUTPUT
<match jfrog.metrics.**>
  @type jfrog_send_metrics
  target_platform "NEWRELIC"
  apikey "#{ENV['NEWRELIC_LICENSE_KEY']}"
  url "#{ENV['NEWRELIC_METRICS_URI']}"
</match>

# ALL CALLHOME
<source>
  @type exec
  tag callhome
  command "curl --request GET '#{ENV['JPD_URL']}/xray/api/v1/system/version' -H 'Authorization: Bearer #{ENV['JPD_ADMIN_TOKEN']}'"
  run_interval 24h
  <parse>
    @type json
  </parse>
</source>
<match callhome>
  @type copy
  <store>
    @type relabel
    @label @jfrogcalhome
  </store>
  <store>
    @type relabel
    @label @heapcallhome
  </store>
</match>
<label @heapcallhome>
  <filter callhome>
    @type record_transformer
    renew_record true
    keep_keys 'productId,features'
    enable_ruby true
    <record>
      app_id '251985503'
      identity "#{ENV['JPD_URL']}"
      event 'newrelic-xray-fluentd-config'
      properties ${return({"featureId" => "XrayVersion/" + record["xray_version"], "productId" => "OnPremObservability-Newrelic/1.0.0"})}
    </record>
  </filter>
  <match callhome>
    @type http
    endpoint "https://heapanalytics.com/api/track"
    open_timeout 5
    content_type application/json
    <format>
      @type json
    </format>
    <buffer>
      flush_interval 5s
    </buffer>
  </match>
</label>
<label @jfrogcalhome>
  <filter callhome>
    @type record_transformer
    renew_record true
    keep_keys 'productId,features'
    enable_ruby true
    <record>
      productId 'OnPremObservability-Newrelic/1.0.0'
      features ${return([{"featureId" => "XrayVersion/" + record["xray_version"]}])}
    </record>
  </filter>
  <match callhome>
    @type http
    endpoint "#{ENV['JPD_URL']}/artifactory/api/system/usage"
    open_timeout 5
    content_type application/json
    <auth>
      method basic
      username "#{ENV['JPD_ADMIN_USERNAME']}"
      password "#{ENV['JPD_ADMIN_TOKEN']}"
    </auth>
    <format>
      @type json
    </format>
    <buffer>
      flush_interval 5s
    </buffer>
  </match>
</label>

# XRAY VIOLATIONS CONFIG
<source>
  @type jfrog_siem
  tag jfrog.xray.siem.vulnerabilities
  jpd_url "#{ENV['JPD_URL']}"
  username "#{ENV['JPD_ADMIN_USERNAME']}"
  token "#{ENV['JPD_ADMIN_TOKEN']}"
  pos_file_path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/jfrog_siem.log.pos"
</source>
<filter jfrog.xray.siem.*>
  @type record_transformer
  enable_ruby true
  <record>
    log_source "jfrog.xray.siem.vulnerabilities"
    category ${record["type"]}
    url ${record["violation_details_url"]}
    signature ${record["watch_name"]}
    cvss ${record["cvss_score"]}
    cve ${record["cve"]}
    vendor_product "Xray"
  </record>
</filter>
<filter jfrog.xray.siem.*>
  @type record_modifier
  <record>
    _impacted_artifacts_url_ ${if record.has_key?('impacted_artifacts_url'); record.delete('impacted_artifacts_url') ; end; nil}
    _type_ ${if record.has_key?('type'); record.delete('type') ; end; nil}
    _watch_name_ ${if record.has_key?('watch_name'); record.delete('watch_name') ; end; nil}
    _violation_details_url_ ${if record.has_key?('violation_details_url'); record.delete('violation_details_url') ; end; nil}
    _cvss_score_ ${if record.has_key?('cvss_score'); record.delete('cvss_score') ; end; nil}
    _cvss_version_ ${if record.has_key?('cvss_version'); record.delete('cvss_version') ; end; nil}
  </record>
  remove_keys _impacted_artifacts_url_, _type_, _watch_name_, _violation_details_url_, _cvss_score_, _cvss_version_
</filter>
<match jfrog.xray.siem.**>
  @type newrelic
  license_key "#{ENV['NEWRELIC_LICENSE_KEY']}"
  logtype "jfrog_xray_violations"
</match>

# SOURCE DIRECTIVES
# SERVICE LOGS
<source>
  @type tail
  @id xray_server_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-server-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-server-service.log.pos"
  tag jfrog.xray.server.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id xray_persist_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-persist-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-persist-service.log.pos"
  tag jfrog.xray.persist.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id xray_indexer_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-indexer-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-indexer-service.log.pos"
  tag jfrog.xray.indexer.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id xray_analysis_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-analysis-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-analysis-service.log.pos"
  tag jfrog.xray.analysis.service
  <parse>
    @type none
  </parse>
</source>
<source>
  @type tail
  @id xray_router_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-service.log.pos"
  tag jfrog.xray.router.service
  <parse>
    @type none
  </parse>
</source>
# TRAEFIK LOGS
<source>
  @type tail
  @id xray_router_traefik_tail
  path "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-traefik.log.pos"
  tag jfrog.xray.router.traefik
  <parse>
      @type regexp
      expression ^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] -(?<message>.+)$
      time_key log_timestamp
      time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>
# REQUEST LOGS
<source>
  @type tail
  @id xray_router_request_tail
  path  "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log"
  pos_file "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/router-request.log.pos"
  tag jfrog.xray.router.request
  <parse>
    @type json
    time_key time
    time_format %Y-%m-%dT%H:%M:%S%:z
  </parse>
</source>
<source>
  @type tail
  @id xray_request_tail
  path  "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-request.log"
  pos_file  "#{ENV['JF_PRODUCT_DATA_INTERNAL']}/log/xray-request.log.pos"
  tag jfrog.xray.xray.request
  <parse>
    @type regexp
    expression ^(?<log_timestamp>[^ ]*)(?<trace_id>[^\|]*)\|(?<remote_address>[^\|]*)\|(?<username>[^\|]*)\|(?<request_method>[^\|]*)\|(?<request_url>[^\|]*)\|(?<return_status>[^\|]*)\|(?<request_content_length>[^\|]*)\|(?<response_content_length>[^\|]*)\|(?<request_duration>[^\|]*)\|(?<request_user_agent>.+)$
    time_key log_timestamp
    time_format %Y-%m-%dT%H:%M:%S.%LZ
  </parse>
</source>


# FILTER DIRECTIVE
## ALL LOGS
<filter **>
  @type concat
  key message
  multiline_start_regexp /\d{4}-\d{1,2}-\d{1,2}/
  timeout_label @NORMAL
  flush_interval 5
</filter>
<match **>
  @type relabel
  @label @NORMAL
</match>
<label @NORMAL>
## SERVICE LOGS
  <filter jfrog.xray.**.service>
    @type record_transformer
    enable_ruby true
    <record>
      message ${record["message"].gsub(/\e\[([;\d]+)?m/, '')}
    </record>
  </filter>
  <filter jfrog.xray.persist.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
    emit_invalid_record_to_error false
  </filter>
  <filter jfrog.xray.indexer.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
    emit_invalid_record_to_error false
  </filter>
  <filter jfrog.xray.analysis.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  <filter jfrog.xray.router.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] \[\] -(?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  <filter jfrog.xray.server.service>
    @type parser
    key_name message
    <parse>
        @type multiline
        format_firstline /\d{4}-\d{1,2}-\d{1,2}/
        format1 /^(?<log_timestamp>[^ ]*) \[(?<service_type>[^\]]*)\] \[(?<log_level>[^\]]*)\] \[(?<trace_id>[^\]]*)\] \[(?<class_line_number>.*)\] \[(?<thread>.*)\] (?<message>.*)$/
        time_key log_timestamp
        time_format %Y-%m-%dT%H:%M:%S.%LZ
    </parse>
  </filter>
  ## ALL LOGS
  <filter jfrog.xray.**>
    @type record_transformer
    <record>
      hostname "#{Socket.gethostname}"
      log_source ${tag}
    </record>
  </filter>

  # TEST OUTPUT
  #<match **>
  #  @type stdout
  #</match>

  # NEWRELIC OUTPUT
  ## NEWRELIC XRAY LOGS OUTPUT
  <match jfrog.**>
    @type newrelic
    license_key "#{ENV['NEWRELIC_LICENSE_KEY']}"
    base_uri "#{ENV['NEWRELIC_LOGS_URI']}"
    logtype "jfrog_xray_logs"
  </match>

</label>