Shell Scripting — Logging & Alerting

#!/usr/bin/env bash
# lib/logging.sh — Production logging library

# ── Configuration ─────────────────────────────────────────
LOG_FILE="${LOG_FILE:-/var/log/myapp/app.log}"
LOG_LEVEL="${LOG_LEVEL:-INFO}"   # DEBUG INFO WARN ERROR
LOG_TO_SYSLOG="${LOG_TO_SYSLOG:-false}"

# ── Level hierarchy ───────────────────────────────────────
declare -A _LOG_LEVELS=([DEBUG]=0 [INFO]=1 [WARN]=2 [ERROR]=3)

_should_log() {
  local msg_level="${1}"
  [[ ${_LOG_LEVELS[$msg_level]} -ge ${_LOG_LEVELS[$LOG_LEVEL]} ]]
}

_log() {
  local level="${1}"; shift
  _should_log "${level}" || return 0
  local ts; ts=$(date '+%Y-%m-%d %H:%M:%S')
  local msg="[${ts}] [${level}] [$$] $*"
  echo "${msg}" | tee -a "${LOG_FILE}"
  [[ "${LOG_TO_SYSLOG}" == "true" ]] && logger -t myapp "${level}: $*"
}

log_debug() { _log "DEBUG" "$@"; }
log_info()  { _log "INFO"  "$@"; }
log_warn()  { _log "WARN"  "$@"; }
log_error() { _log "ERROR" "$@" >&2; }
log_fatal() { _log "ERROR" "$@" >&2; exit 1; }

# /etc/logrotate.d/myapp
/var/log/myapp/*.log {
    daily                    # rotate daily
    rotate 14                # keep 14 rotated files
    compress                 # gzip old files
    delaycompress            # compress yesterday's, not today's
    missingok                # don't error if log is missing
    notifempty               # don't rotate empty files
    create 0640 vriddh adm  # new file permissions owner group
    dateext                  # date suffix instead of numbers
    postrotate
        systemctl reload myapp 2>/dev/null || true
    endscript
}

# Test your logrotate config
logrotate -d /etc/logrotate.d/myapp    # dry run
logrotate -f /etc/logrotate.d/myapp    # force rotation now

# ── Simple mail alert ─────────────────────────────────────
send_alert() {
  local subject="${1}"
  local body="${2}"
  local to="${ALERT_EMAIL:-ops@example.com}"

  echo "${body}" | mail -s "[ALERT] ${subject}" "${to}"
}

# ── Alert with log file attachment ────────────────────────
send_alert_with_log() {
  local subject="${1}"
  local logfile="${2}"
  mail -s "[ALERT] ${subject}" \
       -a "${logfile}" \
       "${ALERT_EMAIL:-ops@example.com}" \
    < "${logfile}"
}

# ── Slack webhook alert ───────────────────────────────────
slack_alert() {
  local message="${1}"
  local colour="${2:-danger}"   # good warning danger
  local webhook="${SLACK_WEBHOOK_URL:?SLACK_WEBHOOK_URL not set}"

  curl -sf -X POST "${webhook}" \
    -H 'Content-Type: application/json' \
    -d "{
      \"attachments\": [{
        \"color\": \"${colour}\",
        \"title\": \"$(hostname -s) Alert\",
        \"text\": \"${message}\",
        \"footer\": \"$(date '+%Y-%m-%d %H:%M:%S')\"
      }]
    }" >/dev/null
}

# Usage in scripts
if (( disk_pct > 90 )); then
  log_error "Disk ${disk_pct}% full on $(hostname)"
  slack_alert "🚨 Disk ${disk_pct}% full on $(hostname)" "danger"
  send_alert "Disk Almost Full - $(hostname)" \
    "Disk usage: ${disk_pct}%\nMount: ${mount_point}"
fi