Icinga2

Go to the Icinga Chair Website.

All configuration files are under the directory /etc/icinga2/conf.d
The configuration is done in three parts.

1. Setup the host information: vms.conf / servers.conf
2. Manage the services and how they are checked (check attributes): services.conf
3. Low level execution of scripts and commands: commands.conf

Normally it is enough to fill out the host information. The services are automatically applied to any hosts that fit the scheme. The Nagios Monitoring scripts are located under
/usr/lib/nagios/plugins/.

These things get checked: ping, ssh, cpu-load, disk-usage
Client

1. Install the nagios plugin package

   sudo apt-get install nagios-plugins-basic

2. Create a new user icinga2

   sudo adduser --disabled-password --gecos "" icinga2

3. Setup SSH Public Key authentication

   sudo mkdir /home/icinga2/.ssh
   sudo vim /home/icinga2/.ssh/authorized_keys 

4. Add following part to the authorized_keys file:

   authorized_keys - vm

   command="/usr/lib/nagios/plugins/check_disk `echo $SSH_ORIGINAL_COMMAND | grep '^-w [0-9,%]* -c [0-9,%]*'` -A -I /sys/kernel/debug/* -I /var/lib/docker/* -I /run/docker/*",no-port-forwarding,no-x11-forwarding,no-agent-forwarding ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC9jOn1brQW5uItt8By/JWaoSnrOU9uruHAIXpIBX1rb6xunezkr4vO7AkppnK9AkbeYqGn2YvKZMFim8UW+wgKoO6jFIbp3ha5uyoL8R2lzRYmnKOkCBRZIJfUjlKUcPXTNgGmYl6QUx20DM2vfcoJ070bK8LoEO06nEddmYQ6RQcm4jO4uUOxdRgQ2WEF+F9fg5R3Qyff2bvLTH0QUqWcgnZu1SPhF+Xzm4PTHpx+d3RR35V5uArhjGmGV6ZPaCpKKPUUtFNx4rTwsQm9z0zUK0r7TyXGQf3+s5ybdMKws8dyGhTPETYxqD1wrcJzw+bhBdvnnmw+OfxERCeO0U2RoO4aJS+CKLJF6l1urKY051TDf3QfLvJNPdy2kE7D4pNSPlj8TJ0FyFlZ8trRYKTdpJekQRYCSwlQmpp2Q/VkgtcdomIb4vPHbxWOaM1crMCAxzSAptVqVjND1ouZA8jtcT9xP8YeG1B2lUQPqwRHtn/nM9rfwrvgQ02WLgYua9QnLmoK6xExIrt5wI44ez3wTBnQSWnMrYrcz1P8YlK4s8rqHeFICulIOsW6h36aP2ijuBkjr/p9sXn/Ki4OgJ5E/lNT90P+AaaYXQwpJocubutPiXgb2YIjnOciltk6BhsboP2xVbW7VheSRsJjkCgiRvzXELG66vAecYjRCkoIPQ== icinga2@icinga
   command="/usr/lib/nagios/plugins/check_load `echo $SSH_ORIGINAL_COMMAND | grep '^-w [0-9,]* -c [0-9,]*'`",no-port-forwarding,no-x11-forwarding,no-agent-forwarding ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDLHbR0NfP5Bas921FK+qcOCLIH716F8dZ6gsO85Ot2NYxKtDIuNEER9h6p3IYGcLLGlWNELyvc4B6O7ityKRA5PLPvxD5XbVp/CjKsdBMMfDJHVD9g9ZW8fCIDDabxpxHGa227fTJMcgXL7VM7qkIx8Sn619bpSGk5MQzv942YqOJ+JrT/11OgERRmqBaXCWAtnX7cyaPUgUDgeR0hmGddyUhOlJvgkX34V8WGNwttJ4vRu/oGqZ0TKPQfKHLsnBzRAwZpEILxGKool0e1VnWuAvkCIK/wVcOxB1y6FKYyYxLarMshwEEzaiE8eaWbwwaGTj4ejftA/rBPZAGUydshWMtuBzIDDXXD2t+Xt9iDS2HCDTTbucn36JbecZ+pdtKoTW9Wo3PbzqPqmJtNEQizIIf+1dboM1MP1eumIGF8XOZpKlTA8+Ola9ItQKMoTMVVCVlTPnk6x8ug2ocX9ykC+12xLpZoaWuzQMbtVyl4C7h1KB1svt5DCCJZDuM1FVwC3wPyUqypZ50dRekwvi+lHRBvSaj0xl/MfQufJiD3wjsj0Y5fSbIKexntvJ/VDq78s2beWFbro8+RN7e09T3Qe2tr3jEQbIBkyfEKdXtLbxVpncT+A6u/QWktO0ZN5g7yFuZbfg+iF21vwgp+2R9IHFkeEqVdYGI5L+0f/qjabQ== icinga2@icinga

   authorized_keys - server

   command="/usr/lib/nagios/plugins/check_disk `echo $SSH_ORIGINAL_COMMAND | grep '^-w [0-9,%]* -c [0-9,%]*'` -A -I /sys/kernel/debug/* -I /var/lib/docker/* -I /run/docker/*",no-port-forwarding,no-x11-forwarding,no-agent-forwarding ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDYSC1aCXTje+8/4fkq5njBoaQwy0WHtspBtYu86KF7yxdat5kTbKnbjwAGDkM+YfMq5u+tf6ehPIffJVI/6pnhBfZ6aTKNqj+VAgD9DBHdiEi+H6p8rmdf480zSAPRTLE6YG/Ca58hotBc23v/99Ud/7ofdWpUCWCLaV5P8SAyLAHfYftyt457BfPCnPkkyYutYKwz02nsNaFtiK3XxGIvvBy3epUuCR+LuZIdg6CcJIZpwu1fGYWGDIbKf/VzUqULoYy/0Zlo9JZlgOdtlNo+6Y404e0qRbVxwrUFUNnEmjCsrVICEmWQoKBQe9T6ShicCURYCcSKEk15ZRYveny62jr2ybC+Tm7Qx94dXb4F8SNLbrsQwjF8mwmyGtYMw1kb1kiuY7+o7zZHJJ6U3gbGf2e21PpP7vrKcquUGO82Gn4LhDbW2BOwuyXUW+2Y8FOR20eFox6er3CeO5rhu9V68aaIchkK5h4+izuHiLKFkEsvSnbuMhCvsWgirjNrSGLZD1W4SbUbFAYq1CP8sBA/dESjstfki4/I2ZZyZ0I2o0erYKiMFpKGEiAD1gc266RVNcD41/rfsb0eFlFI+OZctMINZmr2ZeH4oELinovNF/KWoPvj00wavdxEw/IeeyLz4Jjk7GuYmgVBIeQwUdW7Uo770GkR1eIIh8eZZ1juNQ== icinga2@icinga
   command="/usr/lib/nagios/plugins/check_load `echo $SSH_ORIGINAL_COMMAND | grep '^-w [0-9,]* -c [0-9,]*'`",no-port-forwarding,no-x11-forwarding,no-agent-forwarding ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDdwLRhCohYPnOjyq+QQQ35hTXZJ9/va82IPcQLdE671x+gaGZadOgceO/ymq5ge7dI9ZiBBHeMbI6gqZHTHJCJmQxVcqJ32pRvDzcgi8RnsfLUyQXZulRhFb+Chcaw8lV2R0fJMN5Q4+UhjVNrumx6JDooiinYWEgMBSyo8HIFBs/SWE39Z/FlqDUgyDa9q02TiNkvFBYgLeoDAIyxcdVC6LX8FIjCvlOZLozhQYhmQLzJYhJmpQvIJ4q7SP+zOc08nHHhXqC8h4RvjSImGrAagQd9N5esykaIHbSQglv0bV544Lxms+Fe2MaNKfw3KixNhRvlYmLYSA32tffUWwOLfrKSj96fEeDm9pfL9dLHxJ+1CsvJb1kSwfZoJ71bZg3nu3nQPi4LBpBb8aLiD+e+4mCQGpYZmc9eQ+7iJj5jOucB54HVBHea5DpPZ6foSdJEilq8ziv69uqhWEwQxy8m7MJgKcV/7bNuqID5sbGh893UhJ4mZjRLGsXlEChhA+om+TnBogCYN8UByg/ONpoxYSEqzeJzopMMxa/b4WkJroMr80S+aJKQJnmymP1LMaWkJiLHEsoulmuEGVENM34uFLAb733ChNpVibp0WSCZHoue/CAT9/1wlM3m93z+85yFLHtt1M2XCjky+yYEEQ03mW5zLIzgN6zzOOdPvSFZ5w== icinga2@icinga

5. Change permission of ssh files

   sudo chown -R icinga2:icinga2 /home/icinga2/.ssh/ 

Host

1. Add a new section to the vms.conf/servers.conf

   object Host "new_machine - purpose" {
     address = "<ip-address>"
     vars.os = "Linux"
     check_command = "hostalive"
     vars.type = "disk-vm, load-vm / disk-server, load-server"
     #optional values for disk check
     vars.ssh_command_disk = " -w 50% -c 10%"
     #optional values for load check
     vars.ssh_command_load = " -w 5,4,3 -c 10,6,4"
   }

This section lists the commands to install icinga2. Execute all commands with a prepended sudo or in a administrator shell.

apt-get install software-properties-common
add-apt-repository ppa:formorer/icinga
apt-get update
apt-get install icinga2

Now the commands for icinga2 web:

apt-get install mysql-server mysql-client
#set mysql root password
apt-get install icinga2-ido-mysql
#choose no
mysql -u root -p
mysql> create database icinga; grant all on icinga.* to 'icinga'@'localhost' identified by '<password>';
mysql -u icinga -p icinga < /usr/share/icinga2-ido-mysql/schema/mysql.sql
icinga2 feature enable ido-mysql
icinga2 feature enable command
vim /etc/icinga2/features-enabled/ido-mysql.conf
#fill out the password, user, database fields
service icinga2 restart
------------------------
wget -O - http://packages.icinga.org/icinga.key | apt-key add -
add-apt-repository 'deb http://packages.icinga.org/ubuntu icinga-trusty main'
apt-get update
apt-get install icingaweb2
#some steps because of php7.0
a2dismod mpm_event
a2enmod mpm_prefork
a2enmod php7.0
service apache2 restart
icingacli setup token create
#show token in case you forgot
icingacli setup token show

visit this webpage http://icinga.cm.in.tum.de/icingaweb2/setup

In the settings we change the php timezone to a fixed values

sudo vim /etc/php/7.0/apache2/php.ini
#change this line
date.timezone = "Europe/Berlin"
#install some additional php packages to get graphs working
apt-get install php7.0-intl
apt-get install php7.0-gd
apt-get install php7.0-xml

In the further configuration choose LDAP as the authentification backend:

LDAP RESOURCE
Host: ldap://ldapswitch.informatik.tu-muenchen.de
Port: 389
Root DN: ou=Personen,ou=IN,o=TUM,c=DE
AUTHENTICATION BACKEND
Backend Type: LDAP
Ldap User Object Class: rbgAccount
LDAP User Name Attribute: uid
USER GROUP BACKEND
LDAP Group Object Class: posixGroup
ldap Group Filter : |(gidNumber=5440)(gidNumber=13457)
LDAP Group Name Attribute: cn
LDAP Group Member Attribute: memberUid
LDAP Base DN: ou=Gruppen,ou=IN,ou=TUM,c=DE

Now you need to configure the database access for icingaweb2. Just put in all the information and passwords you got from the step above while installing the icinga2 main component. If you got some weird permission errors resolve them:

chown -R www-data:icingaweb2 /etc/icingaweb2/modules

Set up ip routes so the il11 network (edison network, wifi) is reachable. This is used to monitor devices in the il11 network (e.g. edison sensor devices). On the il11 gateway (vmott3) the firewall has to be set up accordingly to allow commands from the icinga host through the gateway to the network devices.

sudo ip route add 172.24.21.192/27

Add this code to the interfaces file to persist virtual machine reboots.

 sudo vim /etc/network/interfaces
iface ens160 inet dhcp
    up ip route add 172.24.21.192/27 via 131.159.24.141 || true 

1. Address parameter and hostalive check_command has to be set in servers.conf / vms.conf

   object Host "machine" {
     address = "131.159.24.1"
     check_command = "hostalive"
   } 

1. Address parameter and vars.os has to be set in servers.conf / vms.conf

   object Host "machine" {
     address = "131.159.24.1"
     vars.os = "Linux"
   }

1. vars.type needs a disk-vm or disk-server:

object Host "machine" {
  address = "131.159.24.1"
  vars.type = "disk-vm/disk-server"
  #optional line with disk parameters
  vars.ssh_command_disk = " -w 10% -c 5%"
}

apply Service "ssh_disk_server" {
  import "generic-service"
  check_command = "ssh_disk_server"
  display_name = "disk"

  assign where match("*disk-server*", host.vars.type)
}
apply Service "ssh_disk_vm" {
  import "generic-service"
  check_command = "ssh_disk"
  display_name = "disk"

  assign where match("*disk-vm*", host.vars.type)
} 

object CheckCommand "ssh_disk_server" {
        import "plugin-check-command"

        command = [ PluginDir + "/check_by_ssh" ]

        arguments = {
                "-H" = "$ssh_hostname$"
                "-C" = "$ssh_command_disk$"
                "-l" = "$ssh_remote_user$"
                "-i" = "$ssh_identity$"
                }
        vars.ssh_hostname = "$address$"
        vars.ssh_remote_user = "icinga2"
        vars.ssh_identity = "/home/icinga2/.ssh/id_rsa_server_disk"
        vars.ssh_command_disk = " -w 10% -c 5%"
}
object CheckCommand "ssh_disk" {
        import "plugin-check-command"

        command = [ PluginDir + "/check_by_ssh" ]

        arguments = {
                "-H" = "$ssh_hostname$"
                "-C" = "$ssh_command_disk$"
                "-l" = "$ssh_remote_user$"
                "-i" = "$ssh_identity$"
                }
        vars.ssh_hostname = "$address$"
        vars.ssh_remote_user = "icinga2"
        vars.ssh_identity = "/home/icinga2/.ssh/id_rsa_vm_disk"
        vars.ssh_command_disk = " -w 10% -c 5%"
} 

In systems more recent than 15.04 there is a permission big when checking the filesystem.
DISK CRITICAL - /run/lxcfs/controllers is not accessible: Permission denied
There is a workaround:

sudo chown root:root /usr/lib/nagios/plugins/check_disk
sudo chmod u+s /usr/lib/nagios/plugins/check_disk
sudo chmod o+x /usr/lib/nagios/plugins/check_disk 

1. host needs in vars.type a “load-vm/load-server”:

object Host "machine" {
  address = "131.159.24.1"
  vars.type = "load-vm/load-server"
  #optional line with load parameters
  vars.ssh_command_load = " -w 5,4,3 -c 10,6,4"
}

apply Service "ssh_load_server" {
  import "generic-service"
  check_command = "ssh_load_server"
  display_name = "load"
  
  assign where match("*load-server*", host.vars.type)
} 
apply Service "ssh_load" {
  import "generic-service"
  check_command = "ssh_load"
  display_name = "load"

  assign where match("*load-vm*", host.vars.type)
}

object CheckCommand "ssh_load_server" {
        import "plugin-check-command"

        command = [ PluginDir + "/check_by_ssh" ]

        arguments = {
                "-H" = "$ssh_hostname$"
                "-C" = "$ssh_command_load$"
                "-l" = "$ssh_remote_user$"
                "-i" = "$ssh_identity$"
                }
        vars.ssh_hostname = "$address$"
        vars.ssh_remote_user = "icinga2"
        vars.ssh_identity = "/home/icinga2/.ssh/id_rsa_server_load"
        vars.ssh_command_load = " -w 5,4,3 -c 10,6,4"
}
object CheckCommand "ssh_load" {
        import "plugin-check-command"

        command = [ PluginDir + "/check_by_ssh" ]

        arguments = {
                "-H" = "$ssh_hostname$"
                "-C" = "$ssh_command_load$"
                "-l" = "$ssh_remote_user$"
                "-i" = "$ssh_identity$"
                }
        vars.ssh_hostname = "$address$"
        vars.ssh_remote_user = "icinga2"
        vars.ssh_identity = "/home/icinga2/.ssh/id_rsa_vm_load"
        vars.ssh_command_load = " -w 5,4,3 -c 10,6,4"
}

Check HTTP Website for String and certificates. First install nagios-plugins for check_http:

sudo apt install nagios-plugin

Http Check: http_host (req), http_url (default: /), http_ssl (default: true), http_string (req, string to check for)
Http Certificate Check: http_host (req), http_url (defaul: /), http_cert_exp (default: 14,7, first warn value then critical value days till expiration)

object Host "one02 - edge,phi - webserver - ding" {
  address = "131.159.24.86"
  vars.os = "Linux"
  check_command = "hostalive"
  vars.http["Edge Computing Workshop 2017"] = {
        http_host = "edge17.cm.in.tum.de"
        http_string = "Mobile Networking, Analytics and Edge Computing"
        http_ssl = "true"
  }
  vars.http["EdgeSys Workshop 2018"] = {
        http_host = "edgesys18.cm.in.tum.de"
        http_string = "The 1st International Workshop on Edge Systems"
        http_ssl = "true"
  }

  vars.http_cert["Edge Computing Workshop 2017 - Certificate"] = {
        http_cert_exp = "14,7"
        http_host = "edge17.cm.in.tum.de"
  }
  vars.http_cert["EdgeSys Workshop 2018 - Certificate"] = {
        http_cert_exp = "14,7"
        http_host = "edgesys18.cm.in.tum.de"
  }
}

object ServiceGroup "http" {
  display_name = "HTTP Checks"

  assign where match("http*", service.check_command)
}

object ServiceGroup "http_certificates" {
  display_name = "HTTPS Certificate Checks"

  assign where match("http_cert*", service.check_command)
}

object CheckCommand "http_check" {
        import "plugin-check-command"

        command = [ PluginDir + "/check_http" ]

        arguments = {
                "-H" = "$http_host$"
                "-u" = "$http_url$"
                "-s" = "$http_string$"
                "-S" = {
                  description = "Enable SSL/TLS"
                  set_if = "$http_ssl$"
                }
                "-f" = {
                 value= "$http_redirect$"
                 description="Output when redirected (default: warning)"
                }
        }
        vars.http_url = "/"
        vars.http_redirect = "warning"
        vars.http_ssl = "true"
}
object CheckCommand "http_certificate" {
        import "plugin-check-command"

        command = [ PluginDir + "/check_http" ]

        arguments = {
                "-H" = "$http_host$"
                "-C" = "$http_cert_exp$"
                "-u" = "$http_url$"
        }
        vars.http_cert_exp = "14,7"
        vars.http_url = "/"
}

apply Service for (http_host => config in host.vars.http) {
  import "generic-service"

  check_command = "http_check"
  vars += config
}

apply Service for (http_host => config in host.vars.http_cert) {
  import "generic-service"

  check_command = "http_certificate"
  vars += config
}

Icinga Config Changes

1. Check if configs are correct

   sudo service icinga2 checkconfig 

2. Reload icinga config files

   sudo service icinga2 reload 

Icinga directories
Main directory with config files: