proxy-bluecoat.conf

# vim: syntax=python

# This configuration files goes with the proxy-bluecoat.py script to generate the adequate columns.
# ./proxy-bluecoat.py -l <yourlogfile>
# 
# Please check https://github.com/cvandeplas/ELK-forensics for more information.
# Created by Christophe Vandeplas <christophe@vandeplas.com>

input {
  tcp { 
    type => "proxy_bluecoat" 
    port => 18002
    # Most of the bluecoat proxy files are encoded in UTF-8, however the files mix charactersets. (CP1252 and others). There's no way to solve this problem.
    #codec => line { charset => "CP1252" }
  }
}

filter {
  if [type] == "proxy_bluecoat" {
        # drop comment lines
    if ([message] =~ /^#/) {
      drop{}
    }
    csv {
        columns => ["date", "time", "time_taken", "c_ip", "s_supplier_name", "s_supplier_ip", "sc_status", "s_action", "sc_bytes", "cs_bytes", "cs_method", "cs_uri_scheme", "cs_host", "cs_uri_path", "cs_uri_query", "s_hierarchy", "rs_content_type", "cs_user_agent", "sc_filter_result", "sc_filter_category", "s_ip", "s_sitename", "cs_x_forwarded_for", "s_computername", "cs_version", "r_port"]
        separator => " "
    }

    ####
    # All possible values for the BlueCoat proxy are below. 
    #     c = client
    #    cs = client to server
    #     s = proxy server related
    #    sc = server to client
    #     * = means it's in the Kibana dashboard
    ####
    # timestamp          - time   - unix type timestamp
    # gmttime            - time   - GMT date and time of the user request (DD/MM/YYYY:hh:mm:ss GMT)
    # date               - time   - GMT Date in YYYY-MM-DD format
    # time               - time   - GMT time in HH:MM:SS format
    # localtime          - time   - Local date and time of the user request in format: [DD/MMM/YYYY:hh:mm:ss +nnnn]
    if [timestamp] { 
      date { 
        match => ["timestamp", "YYYY-MM-dd HH:mm:ss" ] 
      } 
    } else if [gmttime] { 
      date { 
        match => ["gmttime", "dd/MM/YYYY:HH:mm:ss' GMT'"] 
        timezone => ['UTC']
      } 
    } else if [localtime] {
      date { match => ["localtime", "[dd/MMM/YYYY:HH:mm:ss Z]"] }
    } else if [date] and [time] {
      mutate { merge => ["date", "time"] }   # merge and join need to be in 2 separate mutates
      mutate { join => ["date", " "] }       # merge and join need to be in 2 separate mutates
      date { 
        match => ["date", "YYYY-MM-dd HH:mm:ss" ] 
        timezone => ['UTC']
      } 
    }

    # c_ip               - ip     - * client IP
    # s_supplier_ip      - ip     - IP address used to contact the upstream host (empty for cache hit)
    # s_ip               - ip     - IP address of the appliance on which the client established its connection
    # cs_x_forwarded_for - ip     - * Request header: X-Forwarded-For
    # TODO - convert above to ips using the logstash template
    if ([s_supplier_ip] and [s_supplier_ip] != "-") { 
      geoip { 
        source => "s_supplier_ip"
      }
    }

    # sc_bytes           - int    - bytes from appliance to client
    # time_taken         - int    - time taken (in milliseconds) to process the request
    # r_port             - int    - port from the outbound server URL
    # s_port             - int    - Port of the appliance on which the client established its connection
    # cs_bytes           - int    - Number of bytes sent from client to appliance
    # duration           - int    - Time taken (in seconds) to process the request
    mutate {
      convert => ["sc_bytes", "integer",
                  "time_taken", "integer",
                  "r_port", "integer",
                  "s_port", "integer",
                  "cs_bytes", "integer",
                  "duration", "integer"
                  ]
    }

    # c_dns              - string - * hostname of the client
    # rs_content_type    - string - * response header content type
    # s_supplier_name    - string - hostname of the upstream host (empty for cache hit)
    # sc_filter_category - string - * category of the content filter
    # cs_uri             - string - url 
    # cs_method          - string - * request method
    # cs_request_line    - string - first line of the client's request
    # cs_user            - string - * qualified user
    # cs_host            - string - * hostname from the client's request URL
    # s_action           - string - * what action did the appliance take?
    # s_icap_status      - string - ICAP response status
    # cs_user_agent      - string - * Requestheader:User-Agent
    if [cs_user_agent] != "" {
      useragent { source => "cs_user_agent" prefix => "user_agent." }
    }
    # cs_cookie          - string - Request header: Cookie
    # s_hierarchy        - string - How and where the object was retrieved in the cache hierarchy.
    # s_computername     - string - Configured name of the appliance
    # cs_uri_query       - string - Query from the 'log' URL.
    # cs_referer         - string - Request header: Referer
    # s_sitename         - string - The service type used to process the transaction
    # cs_uri_path        - string - Path from the 'log' URL. Does not include query.
    # cs_version         - string - Protocol and version from the client's request, e.g. HTTP/1.1
    # sc_filter          - string - Content filtering result: Denied, Proxied or result Observed
    
    # s_icap_info        - string - ICAP response information

    # cleanup
    mutate {
      remove_field => ["message", "host", "date", "time", "timestamp", "gmttime", "localtime"]
    }
  }
}

output {
  if [type] == "proxy_bluecoat" {
    elasticsearch {
      index => "logstash-%{[type]}-%{+YYYY.MM.dd}"
      host => "localhost"
      # custom schema
      #template_overwrite => true
      #template => "/etc/elasticsearch/elasticsearch-template.json"
    }
  }
}