From 8fb82bc89ac08e7c51cc69d38670c043ef5a616d Mon Sep 17 00:00:00 2001 From: Beu Date: Mon, 12 Apr 2021 23:33:30 +0200 Subject: [PATCH] Initial Commit --- NGINX/README.md | 21 +++ NGINX/cert-check | 57 ++++++ NGINX/dns-check | 36 ++++ NGINX/module NGINX.yaml | 326 +++++++++++++++++++++++++++++++++ NGINX/nginx-discovery.sh | 42 +++++ NGINX/userparameter_nginx.conf | 1 + 6 files changed, 483 insertions(+) create mode 100644 NGINX/README.md create mode 100644 NGINX/cert-check create mode 100644 NGINX/dns-check create mode 100644 NGINX/module NGINX.yaml create mode 100644 NGINX/nginx-discovery.sh create mode 100644 NGINX/userparameter_nginx.conf diff --git a/NGINX/README.md b/NGINX/README.md new file mode 100644 index 0000000..df4a972 --- /dev/null +++ b/NGINX/README.md @@ -0,0 +1,21 @@ +## What is the purpose of this template: + +This template extends the "Template App Nginx by Zabbix agent" template by adding : +* Certificate detection and verification +* Verification of the existence of a DNS entry for each "server_name" +* Analysis of virtual host logs (5XX responses) +* Verification of the status code of the roots of each virtual host + +## How to setup: + +### On your Zabbix Server and all your Zabbix Proxies: +copy the `dns-check` and `cert-check` files in the directory "/usr/lib/zabbix/externalscripts/" and make them executables. + +### On your servers with Nginx Installed: +copy the `nginx-discovery.sh` file in the directory "/etc/zabbix/scripts" and make it executable. + +copy the `userparameter_nginx.conf` file in the directory "/etc/zabbix/zabbix_agentd.d/" and restart your zabbix agent + +### On your Zabbix WebUI : + +In "Configurations" -> "Templates", clic on the "Import" button and load the `module NGINX.yaml` file diff --git a/NGINX/cert-check b/NGINX/cert-check new file mode 100644 index 0000000..600e14b --- /dev/null +++ b/NGINX/cert-check @@ -0,0 +1,57 @@ +#!/bin/bash + +host=$1 +port=$2 +sni=$3 +proto=$4 + +if [ -z "$sni" ] +then + servername=$host +else + servername=$sni +fi + +if [ -z "$port" ] +then + port="443" +fi + +if [ -n "$proto" ] +then + starttls="-starttls $proto" +fi + +cert_data=`openssl s_client -servername $servername -host $host -port $port $starttls -prexit /dev/null | sed -n '/BEGIN CERTIFICATE/,/END CERT/p'` +if [ -n "$cert_data" ]; then + Rcert=true + validate_hostname=`echo "$cert_data" | openssl x509 -checkhost $servername 2>/dev/null | grep 'does NOT match certificate'` + if [ -z "$validate_hostname" ]; then + Rhostname=true + else + Rhostname=false + fi + end_date=`echo "$cert_data" | openssl x509 -dates -noout 2>/dev/null | sed -n 's/ *notAfter=*//p'` + if [ -n "$end_date" ]; then + end_date_seconds=`date '+%s' --date "$end_date"` + now_seconds=`date '+%s'` + remaining_days=`echo "($end_date_seconds-$now_seconds)/24/3600" | bc` + if [ "$remaining_days" -lt 0 ]; then + Rdays=0 + else + Rdays=$remaining_days + fi + else + echo '-1' + fi + issue_dn=`echo "$cert_data" | openssl x509 -issuer -noout 2>/dev/null | sed -n 's/ *issuer=*//p'` + if [ -n "$issue_dn" ]; then + Rissuer=`echo $issue_dn | sed -n -e 's/, CN = / - /g' -e 's/.*O = //p'` + else + Rissuer="" + fi +else + Rcert=false +fi + +echo "{ \"cert\": ${Rcert}, \"valid_hostname\": ${Rhostname}, \"remaining_days\": ${Rdays}, \"issuer\": \"${Rissuer}\"}" diff --git a/NGINX/dns-check b/NGINX/dns-check new file mode 100644 index 0000000..1eb7eb5 --- /dev/null +++ b/NGINX/dns-check @@ -0,0 +1,36 @@ +#!/bin/bash + +function get_domain_entry { + IFS='.' read -r -a domain <<< "$1" + NS="8.8.8.8" + count=0 + pendingdomain="" + for i in `printf '%s\n' "${domain[@]}"|tac`; do + count=$((count+1)) + pendingdomain="${i}.${pendingdomain}" + for j in $NS; do + NEXTNS=`dig @${j} +timeout=1 +time=1 +tries=2 +noall +authority +answer ${pendingdomain} NS | awk '{if($4 == "NS") {print $5}}' | tr '\n' ' ' | sed -e '/^;;/d'` + if [ -n "$NEXTNS" ];then + NS=$NEXTNS + break + fi + done + if [ "$count" == "${#domain[@]}" ]; then + for j in $NS; do + rawresult=`dig @${j} +timeout=1 +time=1 +tries=2 +noall +authority +answer ${pendingdomain} A ${pendingdomain} AAAA | sed -e '/^;;/d'` + CNAME=`echo "$rawresult" | awk '{if($4 == "CNAME") {print $5}}' | head -n 1` + ENTRY=`echo "$rawresult" | awk '{if($4 == "A" || $4 == "AAAA") {print $5}}'` + if [ -n "$ENTRY" ]; then + echo "$ENTRY" + break + elif [ -n "$CNAME" ]; then + ENTRY=`get_domain_entry "$CNAME"` + echo "$ENTRY" + break + fi + done + fi + done +} +result=`get_domain_entry $1` +echo $result | tr '\n' ' ' diff --git a/NGINX/module NGINX.yaml b/NGINX/module NGINX.yaml new file mode 100644 index 0000000..c3bf8ab --- /dev/null +++ b/NGINX/module NGINX.yaml @@ -0,0 +1,326 @@ +zabbix_export: + version: '5.2' + date: '2021-04-12T09:50:22Z' + groups: + - + name: Templates + - + name: Templates/Modules + templates: + - + template: 'Template Module NGINX' + name: 'Template Module NGINX' + templates: + - + name: 'Template App Nginx by Zabbix agent' + groups: + - + name: Templates + - + name: Templates/Modules + applications: + - + name: DNS + - + name: Logs + - + name: TLS + discovery_rules: + - + name: 'Nginx Certificates discovery' + key: 'nginx_discovery[certificates]' + delay: 12h + lifetime: 48h + item_prototypes: + - + name: 'Information about {#DOMAIN} certificate' + type: EXTERNAL + key: 'cert-check["{HOST.CONN}",443,"{#DOMAIN}"]' + delay: 24h + history: 1d + trends: '0' + value_type: TEXT + applications: + - + name: 'Zabbix raw items' + - + name: 'Existence of {#DOMAIN} certificate' + type: DEPENDENT + key: 'cert-existence[{#DOMAIN}]' + delay: '0' + history: 7d + trends: '0' + value_type: CHAR + applications: + - + name: TLS + preprocessing: + - + type: JSONPATH + parameters: + - $.cert + - + type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 3d + master_item: + key: 'cert-check["{HOST.CONN}",443,"{#DOMAIN}"]' + trigger_prototypes: + - + expression: '{str(true)}=0' + name: 'No valid {#DOMAIN} certificate' + priority: HIGH + manual_close: 'YES' + - + name: 'Remaining days of {#DOMAIN} certificate' + type: DEPENDENT + key: 'cert-remaining-days[{#DOMAIN}]' + delay: '0' + value_type: FLOAT + units: days + applications: + - + name: TLS + preprocessing: + - + type: JSONPATH + parameters: + - $.remaining_days + - + type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 3d + master_item: + key: 'cert-check["{HOST.CONN}",443,"{#DOMAIN}"]' + trigger_prototypes: + - + expression: '{last()}<15' + name: 'TLS Certificate of {#DOMAIN} expires in less than 15 days' + priority: AVERAGE + manual_close: 'YES' + dependencies: + - + name: 'No valid {#DOMAIN} certificate' + expression: '{Template Module NGINX:cert-existence[{#DOMAIN}].str(true)}=0' + - + name: 'TLS Certificate of {#DOMAIN} have expired' + expression: '{Template Module NGINX:cert-remaining-days[{#DOMAIN}].last()}<1' + - + expression: '{last()}<1' + name: 'TLS Certificate of {#DOMAIN} have expired' + priority: HIGH + manual_close: 'YES' + - + name: 'Issuer of {#DOMAIN} certificate' + type: DEPENDENT + key: 'cert-remaining-issuer[{#DOMAIN}]' + delay: '0' + history: 7d + trends: '0' + value_type: TEXT + applications: + - + name: TLS + preprocessing: + - + type: JSONPATH + parameters: + - $.issuer + - + type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 3d + master_item: + key: 'cert-check["{HOST.CONN}",443,"{#DOMAIN}"]' + trigger_prototypes: + - + expression: '{diff()}=1 and {strlen(#1)}>0' + name: 'The issuer of {#DOMAIN} certificate has changed' + priority: INFO + manual_close: 'YES' + - + name: 'Valid Hostname of {#DOMAIN} certificate' + type: DEPENDENT + key: 'cert-valid_hostname[{#DOMAIN}]' + delay: '0' + history: 7d + trends: '0' + value_type: CHAR + applications: + - + name: TLS + preprocessing: + - + type: JSONPATH + parameters: + - $.valid_hostname + - + type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 3d + master_item: + key: 'cert-check["{HOST.CONN}",443,"{#DOMAIN}"]' + trigger_prototypes: + - + expression: '{str(true)}=0' + name: 'The hostname of the {#DOMAIN} certificate does not match' + priority: AVERAGE + dependencies: + - + name: 'No valid {#DOMAIN} certificate' + expression: '{Template Module NGINX:cert-existence[{#DOMAIN}].str(true)}=0' + - + name: 'HTTPS Status code for {#DOMAIN}' + type: DEPENDENT + key: 'https.request.code[{#DOMAIN}]' + delay: '0' + applications: + - + name: Nginx + preprocessing: + - + type: REGEX + parameters: + - '^HTTP\/.* (\d\d\d)' + - \1 + - + type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1d + master_item: + key: 'https.request[{#DOMAIN}]' + trigger_prototypes: + - + expression: | + {last()}<200 or + {last()}>403 + name: '{#DOMAIN} HTTPS Status code is not normal' + priority: AVERAGE + manual_close: 'YES' + - + name: 'HTTPS Request to {#DOMAIN}' + type: HTTP_AGENT + key: 'https.request[{#DOMAIN}]' + delay: 1h + history: 7d + trends: '0' + value_type: TEXT + applications: + - + name: 'Zabbix raw items' + url: 'https://{#DOMAIN}/' + status_codes: '' + follow_redirects: 'NO' + retrieve_mode: HEADERS + verify_peer: 'YES' + verify_host: 'YES' + - + name: 'Nginx Domains discovery' + key: 'nginx_discovery[domains]' + delay: 12h + lifetime: 48h + item_prototypes: + - + name: 'DNS entry for {#DOMAIN}' + type: EXTERNAL + key: 'dns-check[{#DOMAIN}]' + delay: 1h + trends: '0' + value_type: TEXT + applications: + - + name: DNS + trigger_prototypes: + - + expression: '{strlen()}<1' + name: 'No DNS Entry for {#DOMAIN}' + priority: AVERAGE + - + name: 'HTTP Status code for {#DOMAIN}' + type: DEPENDENT + key: 'http.request.code[{#DOMAIN}]' + delay: '0' + applications: + - + name: Nginx + preprocessing: + - + type: REGEX + parameters: + - '^HTTP\/.* (\d\d\d)' + - \1 + - + type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 1d + master_item: + key: 'http.request[{#DOMAIN}]' + trigger_prototypes: + - + expression: | + {last()}<200 or + {last()}>403 + name: '{#DOMAIN} HTTP Status code is not normal' + priority: AVERAGE + manual_close: 'YES' + dependencies: + - + name: 'No DNS Entry for {#DOMAIN}' + expression: '{Template Module NGINX:dns-check[{#DOMAIN}].strlen()}<1' + - + name: 'HTTP Request to {#DOMAIN}' + type: HTTP_AGENT + key: 'http.request[{#DOMAIN}]' + delay: 1h + trends: '0' + value_type: TEXT + applications: + - + name: 'Zabbix raw items' + url: 'http://{#DOMAIN}/' + status_codes: '' + follow_redirects: 'NO' + retrieve_mode: HEADERS + - + name: 'Nginx Access Logs discovery' + key: 'nginx_discovery[logs]' + delay: 12h + lifetime: 48h + item_prototypes: + - + name: 'Number of 500 errors for {#DOMAIN}' + type: CALCULATED + key: '500errors.count[{#DOMAIN},{#PATH}]' + params: 'count("log[{#PATH},.*\\" 5\d\d ,,100,skip]",1m)' + applications: + - + name: Logs + preprocessing: + - + type: DISCARD_UNCHANGED_HEARTBEAT + parameters: + - 3d + - + name: '500 errors for {#DOMAIN}' + type: ZABBIX_ACTIVE + key: 'log[{#PATH},.*\" 5\d\d ,,100,skip]' + trends: '0' + value_type: LOG + description: 'PATH: {#PATH}' + applications: + - + name: Logs + trigger_prototypes: + - + expression: '{Template Module NGINX:log[{#PATH},.*\" 5\d\d ,,100,skip].nodata(1m)}=0 and {Template Module NGINX:500errors.count[{#DOMAIN},{#PATH}].last()}>={$500.ERROR.RATES}' + name: '{#DOMAIN}: Some 500 errors' + priority: HIGH + manual_close: 'YES' + tags: + - + tag: Web + macros: + - + macro: '{$500.ERROR.RATES}' + value: '10' + description: 'This macro is used as a threshold in 500 errors trigger.' diff --git a/NGINX/nginx-discovery.sh b/NGINX/nginx-discovery.sh new file mode 100644 index 0000000..622e124 --- /dev/null +++ b/NGINX/nginx-discovery.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +SERVERNAMELIST=`cat /etc/nginx/sites-enabled/* | sed -e 's/#.*$//g' | grep -F -e 'server_name ' | sed -e 's/server_name //g' -e 's/;//g' | tr ' ' '\n' | sort -u` +VHOSTCONFIG=`cat /etc/nginx/sites-enabled/* | sed -e 's/#.*$//g' | sed -z 's/\n/RETURN/g' | sed 's/RETURNserver/\nserver/g'` +if [ "$1" == "certificates" ] ; then + + for i in $SERVERNAMELIST; do + if [ "$i" != "_" ] ; then + CONFIG=`echo -e "$VHOSTCONFIG" | grep -F " $i" | sed 's/RETURN/\n/g'` + PARSEDDOMAIN=`echo $i | sed -e 's/\*\./wildcard./g'` + if [ -n "`echo -e "$CONFIG" | grep 'ssl_certificate ' | awk '{print $2}' | sed -e 's/;//g'`" ]; then + ZABBIXOUTPUT="${ZABBIXOUTPUT} {\"{#DOMAIN}\":\"${PARSEDDOMAIN}\"}," + fi + fi + done + +elif [ "$1" == "logs" ] ; then + for i in $SERVERNAMELIST; do + if [ "$i" != "_" ] ; then + CONFIG=`echo -e "$VHOSTCONFIG" | grep -F " $i" | sed 's/RETURN/\n/g'` + PARSEDDOMAIN=`echo $i | sed -e 's/\*\./wildcard./g'` + LOGFILES=`echo -e "$CONFIG" | grep -e "access_log " | grep -E -v "access_log( |\t)*none" | awk '{print $2}' | sed -e 's/;//g'` + for j in $LOGFILES; do + ZABBIXOUTPUT="${ZABBIXOUTPUT} {\"{#DOMAIN}\":\"${PARSEDDOMAIN}\",\"{#PATH}\":\"${j}\"}," + done + fi + done + +elif [ "$1" == "domains" ] ; then + for i in $SERVERNAMELIST; do + if [ "$i" != "_" ] ; then + PARSEDDOMAIN=`echo $i | sed -e 's/\*\./wildcard./g'` + ZABBIXOUTPUT="${ZABBIXOUTPUT} {\"{#DOMAIN}\":\"${PARSEDDOMAIN}\"}," + fi + done + +else + echo -e "Usage:\n$0 certificates -> For certificates discovery\n$0 logs -> For logs files discovery\n$0 domains -> For domains discovery" + exit +fi + +echo "[${ZABBIXOUTPUT::-1}]" diff --git a/NGINX/userparameter_nginx.conf b/NGINX/userparameter_nginx.conf new file mode 100644 index 0000000..83003ff --- /dev/null +++ b/NGINX/userparameter_nginx.conf @@ -0,0 +1 @@ +UserParameter=nginx_discovery[*], /etc/zabbix/scripts/nginx-discovery.sh $1