From 2a5f6621b65eaf414fe17dd2e6ea59451650e128 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Tue, 8 Jun 2021 16:59:10 +0200 Subject: [PATCH] Add monitoring Signed-off-by: Yohann D'ANELLO --- group_vars/all/prometheus_node_exporter.yaml | 3 + group_vars/grafana.yml | 7 + group_vars/nginx.yml | 3 + group_vars/prometheus.yml | 9 + host_vars/monitoring.adm.ynerant.fr.yml | 66 +++++++ hosts | 9 + plays/monitoring.yml | 38 ++++ roles/grafana/handlers/main.yml | 5 + roles/grafana/tasks/main.yml | 100 ++++++++++ roles/grafana/templates/ldap.toml.j2 | 47 +++++ .../templates/update-motd.d/05-service.j2 | 3 + roles/ninjabot/tasks/main.yml | 48 +++++ .../templates/ninjabot/ninjabot.json.j2 | 1 + .../systemd/system/ninjabot.service.j2 | 15 ++ .../templates/update-motd.d/05-service.j2 | 3 + .../prometheus-alertmanager/handlers/main.yml | 5 + roles/prometheus-alertmanager/tasks/main.yml | 14 ++ .../templates/prometheus/alertmanager.yml.j2 | 60 ++++++ .../handlers/main.yml | 5 + .../tasks/main.yml | 23 +++ .../handlers/main.yml | 10 + .../prometheus-nginx-exporter/tasks/main.yml | 33 ++++ .../templates/nginx/status.j2 | 8 + roles/prometheus-node-exporter/files/apt.sh | 51 +++++ .../handlers/main.yml | 5 + roles/prometheus-node-exporter/tasks/main.yml | 45 +++++ roles/prometheus/handlers/main.yml | 5 + roles/prometheus/tasks/main.yml | 42 ++++ .../templates/prometheus/alert.rules.yml.j2 | 187 ++++++++++++++++++ .../templates/prometheus/prometheus.yml.j2 | 42 ++++ .../templates/update-motd.d/05-service.j2 | 3 + 31 files changed, 895 insertions(+) create mode 100644 group_vars/all/prometheus_node_exporter.yaml create mode 100644 group_vars/grafana.yml create mode 100644 group_vars/prometheus.yml create mode 100755 plays/monitoring.yml create mode 100644 roles/grafana/handlers/main.yml create mode 100644 roles/grafana/tasks/main.yml create mode 100644 roles/grafana/templates/ldap.toml.j2 create mode 100755 roles/grafana/templates/update-motd.d/05-service.j2 create mode 100644 roles/ninjabot/tasks/main.yml create mode 100644 roles/ninjabot/templates/ninjabot/ninjabot.json.j2 create mode 100644 roles/ninjabot/templates/systemd/system/ninjabot.service.j2 create mode 100755 roles/ninjabot/templates/update-motd.d/05-service.j2 create mode 100644 roles/prometheus-alertmanager/handlers/main.yml create mode 100644 roles/prometheus-alertmanager/tasks/main.yml create mode 100644 roles/prometheus-alertmanager/templates/prometheus/alertmanager.yml.j2 create mode 100644 roles/prometheus-blackbox-exporter/handlers/main.yml create mode 100644 roles/prometheus-blackbox-exporter/tasks/main.yml create mode 100644 roles/prometheus-nginx-exporter/handlers/main.yml create mode 100644 roles/prometheus-nginx-exporter/tasks/main.yml create mode 100644 roles/prometheus-nginx-exporter/templates/nginx/status.j2 create mode 100755 roles/prometheus-node-exporter/files/apt.sh create mode 100644 roles/prometheus-node-exporter/handlers/main.yml create mode 100644 roles/prometheus-node-exporter/tasks/main.yml create mode 100644 roles/prometheus/handlers/main.yml create mode 100644 roles/prometheus/tasks/main.yml create mode 100644 roles/prometheus/templates/prometheus/alert.rules.yml.j2 create mode 100644 roles/prometheus/templates/prometheus/prometheus.yml.j2 create mode 100755 roles/prometheus/templates/update-motd.d/05-service.j2 diff --git a/group_vars/all/prometheus_node_exporter.yaml b/group_vars/all/prometheus_node_exporter.yaml new file mode 100644 index 0000000..39212fa --- /dev/null +++ b/group_vars/all/prometheus_node_exporter.yaml @@ -0,0 +1,3 @@ +--- +glob_prometheus_node_exporter: + listen_addr: "{{ query('ldap', 'ip', ansible_hostname, 'adm') | ipv4 | first }}" diff --git a/group_vars/grafana.yml b/group_vars/grafana.yml new file mode 100644 index 0000000..ab15f6a --- /dev/null +++ b/group_vars/grafana.yml @@ -0,0 +1,7 @@ +--- +glob_grafana: + root_url: https://grafana.ynerant.fr + icon: crans_icon_white.svg + ldap_base: "{{ glob_ldap.base }}" + ldap_master_ipv4: "{{ glob_ldap.servers[0] }}" + ldap_user_tree: "ou=passwd,{{ glob_ldap.base }}" diff --git a/group_vars/nginx.yml b/group_vars/nginx.yml index b883780..a038952 100644 --- a/group_vars/nginx.yml +++ b/group_vars/nginx.yml @@ -30,3 +30,6 @@ glob_nginx: - "172.16.0.0/16" - "fd00:0:0:42::/64" deploy_robots_file: false + +glob_prometheus_nginx_exporter: + listen_addr: "{{ query('ldap', 'ip', ansible_hostname, 'adm') | ipv4 | first }}" diff --git a/group_vars/prometheus.yml b/group_vars/prometheus.yml new file mode 100644 index 0000000..ac686ca --- /dev/null +++ b/group_vars/prometheus.yml @@ -0,0 +1,9 @@ +--- +glob_prometheus: {} + +glob_ninjabot: + config: + nick: templier + server: irc.crans.org + port: 6667 + channel: "#/dev/null" diff --git a/host_vars/monitoring.adm.ynerant.fr.yml b/host_vars/monitoring.adm.ynerant.fr.yml index 2eb6f99..e393cdd 100644 --- a/host_vars/monitoring.adm.ynerant.fr.yml +++ b/host_vars/monitoring.adm.ynerant.fr.yml @@ -2,3 +2,69 @@ interfaces: adm: eth0 srv_nat: eth1 + +loc_prometheus: + node: + file: targets_node.json + targets: "{{ groups['server'] | select('match', '^.*\\.adm\\.ynerant\\.fr$') | list | sort }}" + config: + - job_name: servers + file_sd_configs: + - files: + - '/etc/prometheus/targets_node.json' + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - source_labels: [__param_target] + target_label: __address__ + replacement: '$1:9100' + + nginx: + file: targets_nginx.json + targets: + - proxy.adm.ynerant.fr + config: + - job_name: nginx + file_sd_configs: + - files: + - '/etc/prometheus/targets_nginx.json' + relabel_configs: + - source_labels: [__address__] + target_label: instance + - source_labels: [instance] + target_label: __address__ + replacement: '$1:9117' + + blackbox: + file: targets_blackbox.json + targets: + - https://ynerant.fr/ + - https://bibliogram.ynerant.fr/ + - https://element.ynerant.fr/ + - https://gitea.ynerant.fr/ + - https://grafana.ynerant.fr/ + - https://hydrogen.ynerant.fr/ + - https://nextcloud.ynerant.fr/ + - https://mailu.ynerant.fr/ + - http://notls.ynerant.fr/ + - https://reddit.ynerant.fr/ + - https://thelounge.ynerant.fr/ + - https://translate.ynerant.fr/ + - https://kfet.saperlistpopette.fr/ + config: + - job_name: blackbox + file_sd_configs: + - files: + - '/etc/prometheus/targets_blackbox.json' + metrics_path: /probe + params: + module: [http_2xx] # Look for a HTTP 200 response. + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 diff --git a/hosts b/hosts index 802cb3d..c6060b6 100644 --- a/hosts +++ b/hosts @@ -10,12 +10,18 @@ babel4.adm.ynerant.fr babel5.adm.ynerant.fr babel6.adm.ynerant.fr +[blackbox] +monitoring.adm.ynerant.fr + [certbot] proxy.adm.ynerant.fr [debian:children] server +[grafana] +monitoring.adm.ynerant.fr + [nginx:children] reverseproxy @@ -30,6 +36,9 @@ localhost [postfix] mailu.adm.ynerant.fr +[prometheus] +monitoring.adm.ynerant.fr + [reverseproxy] proxy.adm.ynerant.fr diff --git a/plays/monitoring.yml b/plays/monitoring.yml new file mode 100755 index 0000000..18b08fd --- /dev/null +++ b/plays/monitoring.yml @@ -0,0 +1,38 @@ +#!/usr/bin/env ansible-playbook +--- + +# Deploy Prometheus on monitoring server +- hosts: prometheus + vars: + prometheus: "{{ glob_prometheus | default({}) | combine(loc_prometheus | default({})) }}" + alertmanager: "{{ glob_alertmanager | default({}) | combine(loc_alertmanager | default({})) }}" + ninjabot: "{{ glob_ninjabot | default({}) | combine(loc_ninjabot | default({})) }}" + roles: + - prometheus + - prometheus-alertmanager + - ninjabot + +# Deploy Grafana on monitoring server +- hosts: grafana + vars: + grafana: "{{ glob_grafana | default({}) | combine(loc_grafana | default({})) }}" + roles: + - grafana + +- hosts: blackbox + roles: + - prometheus-blackbox-exporter + +# Monitor all hosts +- hosts: server + vars: + prometheus_node_exporter: "{{ glob_prometheus_node_exporter | default({}) | combine(loc_prometheus_node_exporter | default({})) }}" + roles: + - prometheus-node-exporter + +# Export nginx metrics +- hosts: nginx + vars: + prometheus_nginx_exporter: "{{ glob_prometheus_nginx_exporter | default({}) | combine(loc_prometheus_nginx_exporter | default({})) }}" + roles: + - prometheus-nginx-exporter diff --git a/roles/grafana/handlers/main.yml b/roles/grafana/handlers/main.yml new file mode 100644 index 0000000..cbd4ffd --- /dev/null +++ b/roles/grafana/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart grafana + service: + name: grafana-server + state: restarted diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml new file mode 100644 index 0000000..eb3a091 --- /dev/null +++ b/roles/grafana/tasks/main.yml @@ -0,0 +1,100 @@ +--- +- name: Install GPG + apt: + name: gnupg + state: present + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Import Grafana GPG signing key + apt_key: + url: https://packages.grafana.com/gpg.key + state: present + validate_certs: false + register: apt_key_result + retries: 3 + until: apt_key_result is succeeded + +- name: Add Grafana repository + apt_repository: + repo: deb http://mirror.adm.ynerant.fr/grafana/oss/deb stable main + state: present + update_cache: true + +- name: Install Grafana + apt: + name: grafana + state: present + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Configure Grafana + ini_file: + path: /etc/grafana/grafana.ini + section: "{{ item.section }}" + option: "{{ item.option }}" + value: "{{ item.value }}" + mode: 0640 + loop: + - section: server + option: root_url + value: "{{ grafana.root_url }}" + - section: analytics + option: reporting_enabled + value: "false" + - section: analytics + option: check_for_updates + value: "false" + - section: security + option: disable_initial_admin_creation + value: "true" + - section: security + option: cookie_secure + value: "true" + - section: snapshots + option: external_enabled + value: "false" + - section: users + option: allow_sign_up + value: "false" + - section: users + option: allow_org_create + value: "false" + - section: auth.anonymous + option: enabled + value: "true" + - section: auth.anonymous + option: hide_version + value: "true" + - section: auth.basic # Only LDAP auth + option: enabled + value: "false" + - section: auth.ldap + option: enabled + value: "true" + - section: alerting + option: enabled + value: "false" + notify: Restart grafana + +- name: Configure Grafana LDAP + template: + src: ldap.toml.j2 + dest: /etc/grafana/ldap.toml + mode: 0640 + notify: Restart grafana + +- name: Enable and start Grafana + systemd: + name: grafana-server + enabled: true + state: started + daemon_reload: true + +- name: Indicate role in motd + template: + src: update-motd.d/05-service.j2 + dest: /etc/update-motd.d/05-grafana + mode: 0755 diff --git a/roles/grafana/templates/ldap.toml.j2 b/roles/grafana/templates/ldap.toml.j2 new file mode 100644 index 0000000..5d3e9aa --- /dev/null +++ b/roles/grafana/templates/ldap.toml.j2 @@ -0,0 +1,47 @@ +{{ ansible_header | comment }} +# To troubleshoot and get more log info enable ldap debug logging in grafana.ini +# [log] +# filters = ldap:debug + +[[servers]] +# Ldap server host (specify multiple hosts space separated) +host = "{{ grafana.ldap_master_ipv4 }}" +# Default port is 389 or 636 if use_ssl = true +port = 636 +# Set to true if ldap server supports TLS +use_ssl = true +# Set to true if connect ldap server with STARTTLS pattern (create connection in insecure, then upgrade to secure connection with TLS) +start_tls = false +# set to true if you want to skip ssl cert validation +ssl_skip_verify = true +# set to the path to your root CA certificate or leave unset to use system defaults +# root_ca_cert = "/path/to/certificate.crt" +# Authentication against LDAP servers requiring client certificates +# client_cert = "/path/to/client.crt" +# client_key = "/path/to/client.key" + +# Use direct bind +bind_dn = "uid=%s,{{ grafana.ldap_user_tree }}" + +# Useless as we are doing direct bind, +# but without LDAP auth hang +search_filter = "(uid=%s)" +search_base_dns = ["ou=passwd,dc=ynerant,dc=fr"] + +## For Posix or LDAP setups that does not support member_of attribute you can define the below settings +## Please check grafana LDAP docs for examples +group_search_filter = "(&(objectClass=posixGroup)(memberUid=%s))" +group_search_base_dns = ["ou=group,{{ grafana.ldap_base }}"] +group_search_filter_user_attribute = "cn" + +# Specify names of the ldap attributes your ldap uses +[servers.attributes] +name = "givenName" +surname = "sn" +username = "uid" +email = "mail" + +# All LDAP members can edit +[[servers.group_mappings]] +group_dn = "*" +org_role = "Admin" diff --git a/roles/grafana/templates/update-motd.d/05-service.j2 b/roles/grafana/templates/update-motd.d/05-service.j2 new file mode 100755 index 0000000..d38cb7f --- /dev/null +++ b/roles/grafana/templates/update-motd.d/05-service.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/tail +14 +{{ ansible_header | comment }} +> grafana a été déployé sur cette machine. Voir /etc/grafana/. diff --git a/roles/ninjabot/tasks/main.yml b/roles/ninjabot/tasks/main.yml new file mode 100644 index 0000000..83c0870 --- /dev/null +++ b/roles/ninjabot/tasks/main.yml @@ -0,0 +1,48 @@ +--- +- name: Install python3 IRC library + apt: + name: python3-irc + state: present + update_cache: true + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Install Flask for python3 + apt: + name: python3-flask + state: present + update_cache: true + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Clone NinjaBot code + git: + repo: https://gitlab.crans.org/nounous/NinjaBot.git + dest: /var/local/ninjabot + version: master + +- name: Deploy NinjaBot configuration + template: + src: ninjabot/ninjabot.json.j2 + dest: /var/local/ninjabot/ninjabot.json + +- name: Deploy NinjaBot systemd unit + template: + src: systemd/system/ninjabot.service.j2 + dest: /etc/systemd/system/ninjabot.service + mode: 0644 + +- name: Load and activate NinjaBot service + systemd: + name: ninjabot + daemon_reload: true + enabled: true + state: started + +- name: Indicate NinjaBot in motd + template: + src: update-motd.d/05-service.j2 + dest: /etc/update-motd.d/05-ninjabot + mode: 0755 diff --git a/roles/ninjabot/templates/ninjabot/ninjabot.json.j2 b/roles/ninjabot/templates/ninjabot/ninjabot.json.j2 new file mode 100644 index 0000000..d0296ae --- /dev/null +++ b/roles/ninjabot/templates/ninjabot/ninjabot.json.j2 @@ -0,0 +1 @@ +{{ ninjabot.config | to_nice_json(indent=2) }} diff --git a/roles/ninjabot/templates/systemd/system/ninjabot.service.j2 b/roles/ninjabot/templates/systemd/system/ninjabot.service.j2 new file mode 100644 index 0000000..526a296 --- /dev/null +++ b/roles/ninjabot/templates/systemd/system/ninjabot.service.j2 @@ -0,0 +1,15 @@ +{{ ansible_header | comment }} +[Unit] +Description=NinjaBot IRC bot +After=network.target + +[Service] +Type=simple +WorkingDirectory=/var/local/ninjabot +User=nobody +Group=nogroup +ExecStart=/usr/bin/python3 /var/local/ninjabot/ninjabot.py +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/roles/ninjabot/templates/update-motd.d/05-service.j2 b/roles/ninjabot/templates/update-motd.d/05-service.j2 new file mode 100755 index 0000000..0ac4bcd --- /dev/null +++ b/roles/ninjabot/templates/update-motd.d/05-service.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/tail +14 +{{ ansible_header | comment }} +> NinjaBot a été déployé sur cette machine. Voir /var/local/ninjabot/. diff --git a/roles/prometheus-alertmanager/handlers/main.yml b/roles/prometheus-alertmanager/handlers/main.yml new file mode 100644 index 0000000..3ddbf93 --- /dev/null +++ b/roles/prometheus-alertmanager/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart Prometheus Alertmanager + service: + name: prometheus-alertmanager + state: restarted diff --git a/roles/prometheus-alertmanager/tasks/main.yml b/roles/prometheus-alertmanager/tasks/main.yml new file mode 100644 index 0000000..b65a295 --- /dev/null +++ b/roles/prometheus-alertmanager/tasks/main.yml @@ -0,0 +1,14 @@ +--- +- name: Install Prometheus Alertmanager + apt: + update_cache: true + name: prometheus-alertmanager + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Configure Prometheus Alertmanager + template: + src: prometheus/alertmanager.yml.j2 + dest: /etc/prometheus/alertmanager.yml + notify: Restart Prometheus Alertmanager diff --git a/roles/prometheus-alertmanager/templates/prometheus/alertmanager.yml.j2 b/roles/prometheus-alertmanager/templates/prometheus/alertmanager.yml.j2 new file mode 100644 index 0000000..4c10974 --- /dev/null +++ b/roles/prometheus-alertmanager/templates/prometheus/alertmanager.yml.j2 @@ -0,0 +1,60 @@ +{{ ansible_header | comment }} +# See https://prometheus.io/docs/alerting/configuration/ for documentation. + +global: + # The smarthost and SMTP sender used for mail notifications. + smtp_smarthost: 'localhost:25' + smtp_from: 'alertmanager@example.org' + #smtp_auth_username: 'alertmanager' + #smtp_auth_password: 'password' + +# The directory from which notification templates are read. +templates: +- '/etc/prometheus/alertmanager_templates/*.tmpl' + +# The root route on which each incoming alert enters. +route: + # The labels by which incoming alerts are grouped together. For example, + # multiple alerts coming in for cluster=A and alertname=LatencyHigh would + # be batched into a single group. + group_by: ['instance'] # group per instance + + # When a new group of alerts is created by an incoming alert, wait at + # least 'group_wait' to send the initial notification. + # This way ensures that you get multiple alerts for the same group that start + # firing shortly after another are batched together on the first + # notification. + group_wait: 30s + + # When the first notification was sent, wait 'group_interval' to send a batch + # of new alerts that started firing for that group. + group_interval: 5m + + # If an alert has successfully been sent, wait 'repeat_interval' to + # resend them. + repeat_interval: 24h + + # A default receiver + receiver: webhook-ninjabot + + +# Inhibition rules allow to mute a set of alerts given that another alert is +# firing. +# We use this to mute any warning-level notifications if the same alert is +# already critical. +inhibit_rules: +- source_match: + severity: 'critical' + target_match: + severity: 'warning' + # Apply inhibition if the alertname is the same. + equal: ['alertname', 'cluster', 'service'] + + +receivers: +- name: 'webhook-ninjabot' + webhook_configs: + - url: 'http://localhost:5000/' + send_resolved: true + - url: 'http://localhost:8000/' + send_resolved: true diff --git a/roles/prometheus-blackbox-exporter/handlers/main.yml b/roles/prometheus-blackbox-exporter/handlers/main.yml new file mode 100644 index 0000000..72a6fc5 --- /dev/null +++ b/roles/prometheus-blackbox-exporter/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart prometheus-blackbox-exporter + service: + name: prometheus-blackbox-exporter + state: restarted diff --git a/roles/prometheus-blackbox-exporter/tasks/main.yml b/roles/prometheus-blackbox-exporter/tasks/main.yml new file mode 100644 index 0000000..a571e90 --- /dev/null +++ b/roles/prometheus-blackbox-exporter/tasks/main.yml @@ -0,0 +1,23 @@ +--- +- name: Install Prometheus Blackbox exporter + apt: + update_cache: true + name: prometheus-blackbox-exporter + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Make Prometheus Blackbox exporter listen on localhost only + lineinfile: + path: /etc/default/prometheus-blackbox-exporter + regexp: '^ARGS=' + line: > + ARGS='--config.file /etc/prometheus/blackbox.yml + --web.listen-address="localhost:9115"' + notify: Restart prometheus-blackbox-exporter + +- name: Activate prometheus Blackbox exporter service + systemd: + name: prometheus-blackbox-exporter + enabled: true + state: started diff --git a/roles/prometheus-nginx-exporter/handlers/main.yml b/roles/prometheus-nginx-exporter/handlers/main.yml new file mode 100644 index 0000000..23b542b --- /dev/null +++ b/roles/prometheus-nginx-exporter/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: Restart nginx + service: + name: nginx + state: restarted + +- name: Restart prometheus-nginx-exporter + service: + name: prometheus-nginx-exporter + state: restarted diff --git a/roles/prometheus-nginx-exporter/tasks/main.yml b/roles/prometheus-nginx-exporter/tasks/main.yml new file mode 100644 index 0000000..0fb8001 --- /dev/null +++ b/roles/prometheus-nginx-exporter/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- name: Install prometheus-nginx-exporter + apt: + update_cache: true + name: + - nginx # Nginx may be not already installed + - prometheus-nginx-exporter + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Make prometheus-nginx-exporter listen on adm only + lineinfile: + path: /etc/default/prometheus-nginx-exporter + regexp: '^ARGS=' + line: | + ARGS="-web.listen-address={{ prometheus_nginx_exporter.listen_addr }}:9117 -nginx.scrape-uri=http://[::1]:6424/stub_status" + notify: + - Restart nginx + - Restart prometheus-nginx-exporter + +- name: Configure nginx + template: + src: nginx/status.j2 + dest: /etc/nginx/sites-available/status + notify: Restart nginx + +- name: Activate nginx site + file: + src: /etc/nginx/sites-available/status + dest: /etc/nginx/sites-enabled/status + state: link + notify: Restart nginx diff --git a/roles/prometheus-nginx-exporter/templates/nginx/status.j2 b/roles/prometheus-nginx-exporter/templates/nginx/status.j2 new file mode 100644 index 0000000..b9ed48c --- /dev/null +++ b/roles/prometheus-nginx-exporter/templates/nginx/status.j2 @@ -0,0 +1,8 @@ +{{ ansible_header | comment }} + +server { + listen [::1]:6424; + location = /stub_status { + stub_status; + } +} diff --git a/roles/prometheus-node-exporter/files/apt.sh b/roles/prometheus-node-exporter/files/apt.sh new file mode 100755 index 0000000..7333a67 --- /dev/null +++ b/roles/prometheus-node-exporter/files/apt.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Description: Expose metrics from apt updates. +# +# Author: Ben Kochie + +upgrades="$(/usr/bin/apt-get --just-print dist-upgrade \ + | /usr/bin/awk -F'[()]' \ + '/^Inst/ { sub("^[^ ]+ ", "", $2); gsub(" ","",$2); + sub("\\[", " ", $2); sub("\\]", "", $2); print $2 }' \ + | /usr/bin/sort \ + | /usr/bin/uniq -c \ + | awk '{ gsub(/\\\\/, "\\\\", $2); gsub(/\"/, "\\\"", $2); + gsub(/\[/, "", $3); gsub(/\]/, "", $3); + print "apt_upgrades_pending{origin=\"" $2 "\",arch=\"" $NF "\"} " $1}' +)" + +autoremove="$(/usr/bin/apt-get --just-print autoremove \ + | /usr/bin/awk '/^Remv/{a++}END{printf "apt_autoremove_pending %d", a}' +)" + +orphans="$(comm -23 \ + <(dpkg-query -W -f '${db:Status-Abbrev}\t${Package}\n' \ + | grep '^.[^nc]' | cut -f2 | sort) \ + <(apt-cache dumpavail | sed -rn 's/^Package: (.*)/\1/p' | sort -u) \ + | awk 'END{printf "apt_orphans %d", NR}' +)" + +echo '# HELP apt_upgrades_pending Apt package pending updates by origin.' +echo '# TYPE apt_upgrades_pending gauge' +if [[ -n "${upgrades}" ]] ; then + echo "${upgrades}" +else + echo 'apt_upgrades_pending{origin="",arch=""} 0' +fi + +echo '# HELP apt_autoremove_pending Apt package pending autoremove.' +echo '# TYPE apt_autoremove_pending gauge' +echo "${autoremove}" + +echo '# HELP apt_orphans Orphan apt package.' +echo '# TYPE apt_orphans gauge' +echo "${orphans}" + +echo '# HELP node_reboot_required Node reboot is required for software updates.' +echo '# TYPE node_reboot_required gauge' +if [[ -f '/run/reboot-required' ]] ; then + echo 'node_reboot_required 1' +else + echo 'node_reboot_required 0' +fi diff --git a/roles/prometheus-node-exporter/handlers/main.yml b/roles/prometheus-node-exporter/handlers/main.yml new file mode 100644 index 0000000..b4b64a4 --- /dev/null +++ b/roles/prometheus-node-exporter/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart prometheus-node-exporter + service: + name: prometheus-node-exporter + state: restarted diff --git a/roles/prometheus-node-exporter/tasks/main.yml b/roles/prometheus-node-exporter/tasks/main.yml new file mode 100644 index 0000000..bdb43fc --- /dev/null +++ b/roles/prometheus-node-exporter/tasks/main.yml @@ -0,0 +1,45 @@ +--- +- name: Install Prometheus node-exporter + apt: + update_cache: true + name: prometheus-node-exporter + install_recommends: false # Do not install smartmontools + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Install Prometheus node-exporter-collectors (bullseye) + apt: + update_cache: true + name: prometheus-node-exporter-collectors + install_recommends: false + register: apt_result + retries: 3 + until: apt_result is succeeded + when: + - ansible_lsb.codename == 'bullseye' + +- name: Make Prometheus node-exporter listen on adm only + lineinfile: + path: /etc/default/prometheus-node-exporter + regexp: '^ARGS=' + line: | + ARGS="--web.listen-address={{ prometheus_node_exporter.listen_addr }}:9100" + tags: restart-node-exporter + +- name: Activate prometheus-node-exporter service + systemd: + name: prometheus-node-exporter + enabled: true + state: started + +# Install new APT textfile collector, it might be upstreamed one day +# https://github.com/prometheus-community/node-exporter-textfile-collector-scripts/pull/35 +- name: Patch APT textfile collector + copy: + src: apt.sh + dest: /usr/share/prometheus-node-exporter/apt.sh + owner: root + group: root + mode: 0755 + when: ansible_distribution_release != "bullseye" diff --git a/roles/prometheus/handlers/main.yml b/roles/prometheus/handlers/main.yml new file mode 100644 index 0000000..4214def --- /dev/null +++ b/roles/prometheus/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Restart Prometheus + service: + name: prometheus + state: restarted diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml new file mode 100644 index 0000000..2a9f54f --- /dev/null +++ b/roles/prometheus/tasks/main.yml @@ -0,0 +1,42 @@ +--- +- name: Install Prometheus + apt: + update_cache: true + name: prometheus + register: apt_result + retries: 3 + until: apt_result is succeeded + +- name: Configure Prometheus + template: + src: prometheus/prometheus.yml.j2 + dest: /etc/prometheus/prometheus.yml + mode: 0644 + notify: Restart Prometheus + +- name: Configure Prometheus alert rules + template: + src: prometheus/alert.rules.yml.j2 + dest: /etc/prometheus/alert.rules.yml + mode: 0644 + notify: Restart Prometheus + +# We don't need to restart Prometheus when updating nodes +- name: Configure Prometheus targets + copy: + content: "{{ [{'targets': item.value.targets}] | to_nice_json }}\n" + dest: "/etc/prometheus/{{ item.value.file }}" + mode: 0644 + loop: "{{ prometheus | dict2items }}" + +- name: Activate prometheus service + systemd: + name: prometheus + enabled: true + state: started + +- name: Indicate role in motd + template: + src: update-motd.d/05-service.j2 + dest: /etc/update-motd.d/05-prometheus + mode: 0755 diff --git a/roles/prometheus/templates/prometheus/alert.rules.yml.j2 b/roles/prometheus/templates/prometheus/alert.rules.yml.j2 new file mode 100644 index 0000000..afc37b6 --- /dev/null +++ b/roles/prometheus/templates/prometheus/alert.rules.yml.j2 @@ -0,0 +1,187 @@ +{{ ansible_header | comment }} +{# As this is also Jinja2 it will conflict without a raw block #} +{# Depending of Prometheus Node exporter version, rules can change depending of version #} +{% raw %} +groups: +- name: alert.rules + rules: + + # Alert for any instance that is unreachable for >3 minutes. + - alert: InstanceDown + expr: up == 0 + for: 3m + labels: + severity: critical + annotations: + summary: "{{ $labels.instance }} ({{ $labels.job }}) est invisible depuis plus de 3 minutes !" + + # Alert for out of memory + # Do not take into account memory not used by apps + - alert: OutOfMemory + expr: (node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes + node_memory_PageTables_bytes + node_memory_VmallocUsed_bytes + node_memory_SwapCached_bytes + node_memory_Slab_bytes) / node_memory_MemTotal_bytes * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Mémoire libre de {{ $labels.instance }} à {{ $value }}%." + + # Alert for out of disk space + - alert: OutOfDiskSpace + expr: node_filesystem_free_bytes{fstype="ext4"} / node_filesystem_size_bytes{fstype="ext4"} * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Espace libre de {{ $labels.mountpoint }} sur {{ $labels.instance }} à {{ $value }}%." + + # Alert for out of inode space on disk + - alert: OutOfInodes + expr: node_filesystem_files_free{fstype="ext4"} / node_filesystem_files{fstype="ext4"} * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Presque plus d'inodes disponibles ({{ $value }}% restant) dans {{ $labels.mountpoint }} sur {{ $labels.instance }}." + + # Alert for high CPU usage + - alert: CpuBusy + expr: node_load5 > 9 + for: 10m + labels: + severity: warning + annotations: + summary: "Charge sur {{ $labels.instance }} à {{ $value }}." + + # Check mdadm software RAID + - alert: SoftwareRAIDDegraded + expr: node_md_disks-node_md_disks_active > 0 + for: 3m + labels: + severity: warning + annotations: + summary: "Le RAID sur {{ $labels.instance }} a perdu {{ $value }} disque(s)." + + # Check systemd unit (> buster) + - alert: SystemdServiceFailed + expr: node_systemd_unit_state{state="failed"} == 1 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.name }} a échoué sur {{ $labels.instance }}" + + # Check UPS + - alert: UpsOutputSourceChanged + expr: upsOutputSource != 3 + for: 5m + labels: + severity: warning + annotations: + summary: "La source d'alimentation de {{ $labels.instance }} a changé !" + description: "https://grafana.crans.org/d/qtbg59mZz/alimentation" + + - alert: UpsBatteryStatusChanged + expr: upsBatteryStatus != 2 + for: 5m + labels: + severity: warning + annotations: + summary: "L'état de la batterie de {{ $labels.instance }} a changé !" + description: "https://grafana.crans.org/d/qtbg59mZz/alimentation" + + - alert: UpsTemperatureWarning + expr: (xupsEnvRemoteTemp < 10) or (xupsEnvRemoteTemp > 26) + for: 5m + labels: + severity: warning + annotations: + summary: "La température autour de {{ $labels.instance }} est de {{ $value }}°C." + description: "https://grafana.crans.org/d/qtbg59mZz/alimentation" + + - alert: UpsTemperatureCritical + expr: (xupsEnvRemoteTemp < 0) or (xupsEnvRemoteTemp > 30) + for: 5m + labels: + severity: critical + annotations: + summary: "La température autour de {{ $labels.instance }} est de {{ $value }}°C !" + description: "https://grafana.crans.org/d/qtbg59mZz/alimentation" + + - alert: UpsHighHumidity + expr: xupsEnvRemoteHumidity > 65 + for: 5m + labels: + severity: warning + annotations: + summary: "L'humidité autour de {{ $labels.instance }} est de {{ $value }}%." + description: "https://grafana.crans.org/d/qtbg59mZz/alimentation" + + - alert: UpsVeryHighHumidity + expr: xupsEnvRemoteHumidity > 85 + for: 5m + labels: + severity: critical + annotations: + summary: "L'humidité autour de {{ $labels.instance }} est de {{ $value }}% !" + description: "https://grafana.crans.org/d/qtbg59mZz/alimentation" + + - alert: UpsHighLoad + expr: upsOutputPercentLoad > 70 + for: 5m + labels: + severity: critical + annotations: + summary: "La charge de {{ $labels.instance }} est de {{ $value }}% !" + description: "https://grafana.crans.org/d/qtbg59mZz/alimentation" + + - alert: UpsWrongInputVoltage + expr: (upsInputVoltage < 210) or (upsInputVoltage > 250) + for: 5m + labels: + severity: warning + annotations: + summary: "La tension d'entrée de {{ $labels.instance }} est de {{ $value }}V." + description: "https://grafana.crans.org/d/qtbg59mZz/alimentation" + + - alert: UpsWrongOutputVoltage + expr: (upsOutputVoltage < 215) or (upsOutputVoltage > 245) + for: 5m + labels: + severity: warning + annotations: + summary: "La tension de sortie de {{ $labels.instance }} est de {{ $value }}V." + description: "https://grafana.crans.org/d/qtbg59mZz/alimentation" + + - alert: AptAutoremovePending + expr: apt_autoremove_pending > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "{{ $value }} paquet(s) APT sont inutile(s) sur {{ $labels.instance }}." + + - alert: MailqNotEmpty + expr: postfix_mailq_length > 25 + for: 1m + labels: + severity: warning + annotations: + summary: "{{ $value }} mails dans la mailq sur {{ $labels.instance }}." + + - alert: NoRadiusLogin + expr: rate(radiusd_access_ok[3m]) == 0 + for: 2m + labels: + severity: warning + annotations: + summary: "Personne ne vient taper le RADIUS." + + - alert: TooManyReallocatedSectors + expr: smartmon_reallocated_sector_ct_raw_value > 1e3 + for: 5m + labels: + severity: warning + annotations: + summary: "{{ $labels.disk }} sur {{ $labels.instance }} a {{ $value }} secteurs réalloués." + +{% endraw %} diff --git a/roles/prometheus/templates/prometheus/prometheus.yml.j2 b/roles/prometheus/templates/prometheus/prometheus.yml.j2 new file mode 100644 index 0000000..daa136c --- /dev/null +++ b/roles/prometheus/templates/prometheus/prometheus.yml.j2 @@ -0,0 +1,42 @@ +{{ ansible_header | comment }} + +global: + # scrape_interval is set to the global default (60s) + # evaluation_interval is set to the global default (60s) + # scrape_timeout is set to the global default (10s). + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'example' + +# Alertmanager configuration +# Use prometheus alertmanager installed on the same machine +alerting: + alertmanagers: + - static_configs: + - targets: ['localhost:9093'] + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + - "alert.rules.yml" # Monitoring alerts, this is the file you may be searching! + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +{{ + { + "scrape_configs": + [ + { + "job_name": "prometheus", + "static_configs" : [ + { + "targets": [ + "localhost:9090" + ] + } + ] + } + ] + (prometheus | json_query("*.config[0]")) + } | to_nice_yaml(indent=2) +}} diff --git a/roles/prometheus/templates/update-motd.d/05-service.j2 b/roles/prometheus/templates/update-motd.d/05-service.j2 new file mode 100755 index 0000000..41969b9 --- /dev/null +++ b/roles/prometheus/templates/update-motd.d/05-service.j2 @@ -0,0 +1,3 @@ +#!/usr/bin/tail +14 +{{ ansible_header | comment }} +> prometheus a été déployé sur cette machine. Voir /etc/prometheus/.