redshift-over-s3.conf

input{
s3 {
#Name of S3 bucket
bucket =>"<bucket_name>" 
#AWS region where the Redshift/Bucket is configured
region =>"<region>"
# AWS ACCESS Key required for authentication  
access_key_id =>"<ACCESS_KEY>"  
#AWS Secret Key required for authentication
secret_access_key =>"<SECRET_ACCESS_KEY>" 
# type field, value should be same as used in filter part.  
type =>"redS"  
#Insert the account id of the AWS account
add_field => {"account_id" => "<ACCOUNT_ID>"}
#Insert the name of cluster you have created
add_field => {"cluster_name" => "<CLUSTER_NAME>"}
# Prefix for log location, helps in selecting all logs in the bucket.
prefix =>"<prefix/>"  
# multiline codec helps in concatenating multiline log in single event
codec => multiline {
      pattern => "((^'%{TIMESTAMP_ISO8601:timestamp})|(^(?<action>[^:]*) \|%{DAY:day}\, %{MONTHDAY:md} %{MONTH:month} %{YEAR:year} %{TIME:time}))"
      negate => true
      what => "previous"
    }
}
}filter 
{
if [type] == "redS" {
mutate 
{
add_field => { "serverHostnamePrefix" => "%{account_id}-%{cluster_name}" }
add_field => { "dbprefix" => "%{account_id}:%{cluster_name}" }
}
if "::" in [message] { 
grok {
match => {"message" => ["(?<action>[^:]*) \|%{DAY:day}\, %{MONTHDAY:md} %{MONTH:month} %{YEAR:year} %{TIME:time}\|(?<remotehost>[^\s]*) \|(?<remoteport>[^\s]*) \|(?<pid>[^\s]*)\|(?<dbname>[^\s]*) \|(?<username>[^\s]*) \|(?<authmethod>[^:]*) \|%{NUMBER:duration}\|(?<sslversion>[^\s]*) \|(?<sslcipher>[^:]*) \|(?<mtu>[^\s]*)\|%{SPACE:sslcompression}\|%{SPACE:sslexpansion}\|%{SPACE:iamauthguid}\|(?<application_name>[^:]*) \|(?<os_version>[^:]*) \|(?<driver_version>[^:]*) \|(?<plugin_name>[^:]*) \|(?<protocol_version>[^:]*)\|(?<id>[^:]*)",
"(?<action>[^:]*) \|%{DAY:day}\, %{MONTHDAY:md} %{MONTH:month} %{YEAR:year} %{TIME:time}\|(?<remotehost>[^\s]*) \|(?<remoteport>[^\s]*) \|(?<pid>[^\s]*)\|(?<dbname>[^\s]*) \|(?<username>[^\s]*) \|(?<authmethod>[^:]*) \|(?<duration>[^\s]*)\|(?<sslversion>[^\s]*) \|(?<sslcipher>[^\s]*) \|(?<mtu>[^\s]*)\|%{SPACE:sslcompression}\|%{SPACE:sslexpansion}\|(?<iamauthguid>[^\s]*) \|(?<applicationname>[^:]*) \|(?<os_version>[^:]*) \|(?<driverversion>[^:]*) \|%{SPACE:plugin_name} \|(?<protocolversion>[^:]*)\|(?<id>[^\s]*)"
]
}
}
}
if "LOG:" in [message] {
grok {
match => {"message" => "\'%{TIMESTAMP_ISO8601:timestamp} %{TZ:timezone}\s\[\s(?<dbtype>[^\s]*)=(?<dbname>[^\s]*)\s(?<usertype>[^\s]*)=(?<user>[^\s]*)\s(?<pidtype>[^\s]*)=(?<pid>[^\s]*)\s(?<useridtype>[^\s]*)=(?<userid>[^\s]*)\s(?<xidtype>[^\s]*)=(?<xid>[^\s]*)\s\]\'\s%{GREEDYDATA:sql_query}"
}
}
}
redshift_guardium_connector{}
}
}