logstash Archives | Eli's Blog

Logstash Nagios Configuration Grok example (logstash.conf)

March 14, 2016July 21, 2017 Eli Kleinman 2 Comments

(No Ratings Yet)

Nagios logstash yml conf file

input {
  file {
    # Wildcards work, here :)
    path => [ "/var/log/messages" ]
    start_position => "beginning"
    type => "nagios-alert"
  }
}

#input {
    #tcp {
        #host => "10.10.10.1"
        #port => 3333
        #type => "nagios-alert"
    #}
#}

filter {
  if [type] == "nagios-alert" {
    if [message] =~ /nagios3 nagios:/ {
      if [message] =~ /SERVICE DOWNTIME ALERT/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service_command};%{DATA:nagios_service_stat};%{GREEDYDATA:nagios_message}" }
        }
      }
      if [message] =~ /SERVICE ALERT/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_serivce_up_down};%{DATA:nagios_state};%{DATA:nagios_serivce_stat_up_down};%{DATA:nagios_serivce_retry_up_down};%{GREEDYDATA:nagios_message}" }
        }
      }
      if [message] =~ /HOST DOWNTIME ALERT/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_host_stat};%{GREEDYDATA:nagios_message}" }
        }
      }
      if [message] =~ /HOST ALERT/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_host_up_down};%{DATA:nagios_host_stat_up_down};%{DATA:nagios_host_retry_up_down};%{GREEDYDATA:nagios_message}" }
        }
      }
      if [message] =~ /CURRENT SERVICE STATE/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service_command};%{DATA:nagios_serivce_up_down};%{DATA:nagios_serivce_stat_up_down};%{DATA:nagios_serivce_retry_up_down};%{GREEDYDATA:nagios_message}" }
        }
      }
      if [message] =~ /CURRENT HOST STATE/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_host_up_down};%{DATA:nagios_host_stat_up_down};%{DATA:nagios_host_retry_up_down};%{GREEDYDATA:nagios_message}" }
        }
      }
      if [message] =~ /HOST NOTIFICATION/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}" }
        }
      }
      if [message] =~ /SERVICE NOTIFICATION/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service_command};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}" }
        }
      }
      if [message] =~ /EXTERNAL COMMAND/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_message};%{GREEDYDATA:nagios_hostname}" }
        }
      }
      if [message] =~ /Warning/ {
        grok {
            match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MLogstash nagios conf(YML) fileONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{GREEDYDATA:nagios_message}" }
        }
      }

    if ("" in [MONTH]) {
      mutate {
        # Replace field
        gsub => [ "TIME", ",", "." ]
        add_field => { "mytimestamp" => "%{MONTH} %{MONTHDAY} 2016 %{TIME}" }
      }
      date {
        match => [ "mytimestamp", "MMM dd YYYY HH:mm:ss", "MMM d YYYY HH:mm:ss" ]
        #timezone => "UTC"
        target => "@timestamp"
      }
      mutate {
        remove_field => [ "mytimestamp", "%{MONTH} %{MONTHDAY} %{YEAR} %{TIME}" ]
      }
     }
    }
     metrics {
       meter => "events"
       add_tag => "metric"
       flush_interval => 60
     }
  }
}

output {
    ## Debug
    #stdout { codec => rubydebug }

    if [type] == "nagios-alert" {
      elasticsearch {
        hosts => [ "10.10.3.11:9204", "10.10.3.12:9204", "10.10.3.11:9205", "10.10.3.12:9205" ]
        #hosts => [ "10.10.3.11:9200" ]
        timeout => 30
        index => "nagios-syslog-%{+YYYY.MM.dd}"
        ##flush_size => 2000
        #flush_size => 5
      }
    } else if "metric" in [tags] {
      file {
        codec => line {
          format => "rate: %{[events][rate_1m]}"
          #format => "rate: %{[events][rate_5m]}"
        }
          path => "/var/tmp/logstash-%{+YYYY-MM-dd}.log"
      }
    } else {
      elasticsearch{
        hosts => [ "10.10.3.11:9204", "10.10.3.12:9204", "10.10.3.11:9205", "10.10.3.12:9205" ]
        #hosts => [ "10.10.3.36:9200" ]
        timeout => 30
        #flush_size => 2000
      }
    }
    # For testing only
    #if [type] == "nagios-alert" {
      #null{}
    #}
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

input {

file {

# Wildcards work, here :)

path => [ "/var/log/messages" ]

start_position => "beginning"

type => "nagios-alert"

}

#input {

#tcp {

#host => "10.10.10.1"

#port => 3333

#type => "nagios-alert"

filter {

if [type] == "nagios-alert" {

if [message] =~ /nagios3 nagios:/ {

if [message] =~ /SERVICE DOWNTIME ALERT/ {

grok {

match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service_command};%{DATA:nagios_service_stat};%{GREEDYDATA:nagios_message}" }

}

if [message] =~ /SERVICE ALERT/ {

grok {

match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_serivce_up_down};%{DATA:nagios_state};%{DATA:nagios_serivce_stat_up_down};%{DATA:nagios_serivce_retry_up_down};%{GREEDYDATA:nagios_message}" }

}

if [message] =~ /HOST DOWNTIME ALERT/ {

grok {

}

if [message] =~ /HOST ALERT/ {

grok {

match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_host_up_down};%{DATA:nagios_host_stat_up_down};%{DATA:nagios_host_retry_up_down};%{GREEDYDATA:nagios_message}" }

}

if [message] =~ /CURRENT SERVICE STATE/ {

grok {

}

if [message] =~ /CURRENT HOST STATE/ {

grok {

match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_host_up_down};%{DATA:nagios_host_stat_up_down};%{DATA:nagios_host_retry_up_down};%{GREEDYDATA:nagios_message}" }

}

if [message] =~ /HOST NOTIFICATION/ {

grok {

match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}" }

}

if [message] =~ /SERVICE NOTIFICATION/ {

grok {

match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service_command};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message}" }

}

if [message] =~ /EXTERNAL COMMAND/ {

grok {

match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{DATA:nagios_message};%{GREEDYDATA:nagios_hostname}" }

}

if [message] =~ /Warning/ {

grok {

match => { "message" => "%{MONTH:MONTH}\s+%{MONTHDAY:MLogstash nagios conf(YML) fileONTHDAY} %{TIME:TIME} %{HOSTNAME:HOSTNAME} %{HOSTNAME:SHORTHOSTNAME}: %{DATA:nagios_type}: %{GREEDYDATA:nagios_message}" }

}

if ("" in [MONTH]) {

mutate {

# Replace field

gsub => [ "TIME", ",", "." ]

add_field => { "mytimestamp" => "%{MONTH} %{MONTHDAY} 2016 %{TIME}" }

}

date {

match => [ "mytimestamp", "MMM dd YYYY HH:mm:ss", "MMM d YYYY HH:mm:ss" ]

#timezone => "UTC"

target => "@timestamp"

}

mutate {

remove_field => [ "mytimestamp", "%{MONTH} %{MONTHDAY} %{YEAR} %{TIME}" ]

}

metrics {

meter => "events"

add_tag => "metric"

flush_interval => 60

}

output {

## Debug

#stdout { codec => rubydebug }

if [type] == "nagios-alert" {

elasticsearch {

hosts => [ "10.10.3.11:9204", "10.10.3.12:9204", "10.10.3.11:9205", "10.10.3.12:9205" ]

#hosts => [ "10.10.3.11:9200" ]

timeout => 30

index => "nagios-syslog-%{+YYYY.MM.dd}"

##flush_size => 2000

#flush_size => 5

}

} else if "metric" in [tags] {

file {

codec => line {

format => "rate: %{[events][rate_1m]}"

#format => "rate: %{[events][rate_5m]}"

}

path => "/var/tmp/logstash-%{+YYYY-MM-dd}.log"

}

} else {

elasticsearch{

hosts => [ "10.10.3.11:9204", "10.10.3.12:9204", "10.10.3.11:9205", "10.10.3.12:9205" ]

#hosts => [ "10.10.3.36:9200" ]

timeout => 30

#flush_size => 2000

}

# For testing only

#if [type] == "nagios-alert" {

#null{}

}

Solaris Elasticsearch Forwarder(Logstash) Setup

March 8, 2016July 5, 2017 Eli Kleinman Leave a comment

(No Ratings Yet)

Create elastic data pool

zpool create data1 c1d1

1	zpool create data1 c1d1

Add elastic user

groupadd elastic
useradd -d /export/home/elastic -g elastic -m -s /bin/bash -c "Elastic Search" elastic

1 2	groupadd elastic useradd -d /export/home/elastic -g elastic -m -s /bin/bash -c "Elastic Search" elastic

Install elastic binary’s

cd /opt;unzip -qq /install/elasticsearch-2.2.0.zip;mv elasticsearch-2.2.0 elasticsearch

mkdir /data1 /data1/data /data1/plugins /data1/log
mkdir /opt/elasticsearch/current /opt/elasticsearch/current/logs /opt/elasticsearch/current/data /opt/elasticsearch/current/tmp

chown -R elastic:elastic  /opt/elasticsearch /data1

cd /opt;unzip -qq /install/elasticsearch-2.2.0.zip;mv elasticsearch-2.2.0 elasticsearch

mkdir /data1 /data1/data /data1/plugins /data1/log

mkdir /opt/elasticsearch/current /opt/elasticsearch/current/logs /opt/elasticsearch/current/data /opt/elasticsearch/current/tmp

chown -R elastic:elastic /opt/elasticsearch /data1

Grant access for elastic user

echo "elastic::::lock_after_retries=no;defaultpriv=all,file_dac_read;profiles=Primary Administrator,All" >>/etc/user_attr

1	echo "elastic::::lock_after_retries=no;defaultpriv=all,file_dac_read;profiles=Primary Administrator,All" >>/etc/user_attr

Configre system limits for user elastic /etc/project

elastic:100::::process.max-file-descriptor=(priv,65536,deny);process.max-sem-nsems=(priv,1024,deny);project.max-sem-ids=(priv,256,deny);project.max-shm-memory=(privileged,68719476736,deny)

1	elastic:100::::process.max-file-descriptor=(priv,65536,deny);process.max-sem-nsems=(priv,1024,deny);project.max-sem-ids=(priv,256,deny);project.max-shm-memory=(privileged,68719476736,deny)

add to /opt/elasticsearch/bin/elasticsearch (top)

ulimit -n unlimited

1	ulimit -n unlimited

Configure startup script

svccfg import elasticsearch.xml

1	svccfg import elasticsearch.xml

Configure elasticsearch.yml for hot warm (SSD) Hot warm config Configure hot template

curl -XPUT http://elk2.domain.com:9200/_template/web-sql-log -d '              
{
  "template": "web-sql-log-*",
  "settings": {
    "index.refresh_interval": "5s",
    "index.routing.allocation.require.box_type": "hot",
    "number_of_shards" : 3
  }
}
'
{"acknowledged":true}

curl -XPUT http://elk2.domain.com:9200/_template/web-sql-log -d '

{

"template": "web-sql-log-*",

"settings": {

"index.refresh_interval": "5s",

"index.routing.allocation.require.box_type": "hot",

"number_of_shards" : 3

}

{"acknowledged":true}

Add latter the mappings

  "mappings": {
  "jmx": {
    "properties": {
      "OpenSessions_Count": {
        "type": "long"
      },
      "PendingUserRequestCount_Count": {
        "type": "long"
      },
      "Host_Name": {
        "type": "string",
	"index": "not_analyzed"
      },
      "@timestamp": {
        "format": "strict_date_optional_time||epoch_millis",
        "type": "date"
      },
      "ActiveExecuteThreads_ActvCount": {
        "type": "integer"
      },
      "ActiveExecuteThreads_MaxCount": {
        "type": "integer"
      },
      "@version": {
        "type": "string"
      },
      "host": {
        "type": "string",
	"index": "not_analyzed"
      },
      "ActiveConnectionsCount_Count": {
        "type": "long"
      },
      "message": {
        "type": "string"
      },
      "type": {
        "type": "string"
      },
      "command": {
        "type": "string"
      }
    }
  }

"mappings": {

"jmx": {

"properties": {

"OpenSessions_Count": {

"type": "long"

"PendingUserRequestCount_Count": {

"type": "long"

"Host_Name": {

"type": "string",

"index": "not_analyzed"

"@timestamp": {

"format": "strict_date_optional_time||epoch_millis",

"type": "date"

"ActiveExecuteThreads_ActvCount": {

"type": "integer"

"ActiveExecuteThreads_MaxCount": {

"type": "integer"

"@version": {

"type": "string"

"host": {

"type": "string",

"index": "not_analyzed"

"ActiveConnectionsCount_Count": {

"type": "long"

"message": {

"type": "string"

"type": {

"type": "string"

"command": {

"type": "string"

}

To […]

Solaris Logstash Forwarder Configuration and Setup

February 17, 2016July 21, 2017 Eli Kleinman Leave a comment

(No Ratings Yet)

Click here for the Full Install ELK and Configure Create elastic user and group

groupadd elastic
useradd -d /export/home/elastic -g elastic -m -s /bin/bash -c "Elastic Search" elastic

1 2	groupadd elastic useradd -d /export/home/elastic -g elastic -m -s /bin/bash -c "Elastic Search" elastic

Create elastic user home directory

mkdir /export/home/elastic
chown elastic:elastic /export/home/elastic

1 2	mkdir /export/home/elastic chown elastic:elastic /export/home/elastic

Download logstash gz file and create directory

cd /opt
tar xf  /var/tmp/logstash-2.2.2.tar.gz
mv logstash-2.2.2 logstash

cd /opt

tar xf /var/tmp/logstash-2.2.2.tar.gz

mv logstash-2.2.2 logstash

Modify two places and add solaris

if RbConfig::CONFIG['host_os'].downcase =~ /darwin|openbsd|freebsd|netbsd|linux|solaris/
  require 'java'

  result = begin
    if RbConfig::CONFIG['host_os'].downcase =~ /darwin|openbsd|freebsd|netbsd|solaris/
      require File.join(File.dirname(__FILE__), 'bsd_console'

require 'java'

result = begin

if RbConfig::CONFIG['host_os'].downcase =~ /darwin|openbsd|freebsd|netbsd|solaris/

require File.join(File.dirname(__FILE__), 'bsd_console'

create logstash other directory’s

mkdir /opt/logstash/current 
mkdir /opt/logstash/current/logs /opt/logstash/current/data /opt/logstash/current/tmp

1 2	mkdir /opt/logstash/current mkdir /opt/logstash/current/logs /opt/logstash/current/data /opt/logstash/current/tmp

Create logstash start-up script cat /opt/logstash/bin/start_logstash.sh

cd /opt/logstash/current/logs

ulimit -n 65000
name="logstash"
LS_LOG_DIR=/opt/logstash/current/logs
#LS_CONF_DIR=/opt/logstash/current/conf
LS_LOG_FILE="${LS_LOG_DIR}/$name.log"
LS_CONF=/opt/logstash/current/$name.yml
# On send side
LS_OPTS="-w 4"
# On receive side
# LS_OPTS="-w 12"
program=/opt/logstash/bin/logstash
#args="agent -f ${LS_CONF_DIR} -l ${LS_LOG_FILE} ${LS_OPTS}"
args="agent -f ${LS_CONF} -l ${LS_LOG_FILE} ${LS_OPTS}"

#nohup $program -w $1 -f $2 > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &
exec "$program" $args > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &

chmod 750 /opt/logstash/bin/start_logstash.sh

cd /opt/logstash/current/logs

ulimit -n 65000

name="logstash"

LS_LOG_DIR=/opt/logstash/current/logs

#LS_CONF_DIR=/opt/logstash/current/conf

LS_LOG_FILE="${LS_LOG_DIR}/$name.log"

LS_CONF=/opt/logstash/current/$name.yml

# On send side

LS_OPTS="-w 4"

# On receive side

# LS_OPTS="-w 12"

program=/opt/logstash/bin/logstash

#args="agent -f ${LS_CONF_DIR} -l ${LS_LOG_FILE} ${LS_OPTS}"

args="agent -f ${LS_CONF} -l ${LS_LOG_FILE} ${LS_OPTS}"

#nohup $program -w $1 -f $2 > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &

exec "$program" $args > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &

chmod 750 /opt/logstash/bin/start_logstash.sh

Set proper owner

chown -R elastic:elastic /opt/logstash

1	chown -R elastic:elastic /opt/logstash

Create logstash configuration […]

Tag: logstash

Logstash Nagios Configuration Grok example (logstash.conf)

Solaris Elasticsearch Forwarder(Logstash) Setup

Solaris Logstash Forwarder Configuration and Setup

About This Blog

Contact Us

Meta