This page puts AWK to work on the kinds of log analysis problems that appear daily in production operations. Every example here is a real task from real ops workflows — finding slow endpoints, detecting attack patterns, correlating errors across services, and building the kind of fast one-liner analysis that's worth more than a dashboard during an incident.
1
Nginx/Apache access log analysis
AWK
# Nginx combined log format:
# $remote_addr - $remote_user [$time] "$request" $status $bytes "$referrer" "$ua"
# 10.0.1.5 - - [01/May/2026:10:14:02 +0530] "GET /api/users HTTP/1.1" 200 4821 "-" "curl/8.0"
# ── Request count by status code ──────────────────────────
awk '{ status[$9]++ }
END {
for (s in status)
printf "HTTP %s: %5d requests\n", s, status[s]
}' access.log | sort
# ── Top 10 IPs by request count ───────────────────────────
awk '{ count[$1]++ }
END { for (ip in count) print count[ip], ip }' access.log \
| sort -rn | head -10
# ── Bandwidth by IP ───────────────────────────────────────
awk '$10 ~ /^[0-9]+$/ { bytes[$1] += $10 }
END {
for (ip in bytes) {
mb = bytes[ip] / 1048576
printf "%-18s %8.1f MB\n", ip, mb
}
}' access.log | sort -k2 -rn | head -10
# ── 4xx/5xx error rate per hour ───────────────────────────
awk '$9 ~ /^[45]/ {
# Extract hour from [01/May/2026:10:14:02
split($4, t, ":")
hour = t[2]
errors[hour]++
total[hour]++
}
$9 ~ /^[23]/ {
split($4, t, ":")
total[t[2]]++
}
END {
for (h in total)
printf "Hour %s: %d errors / %d total (%.1f%%)\n",
h, errors[h]+0, total[h], (errors[h]+0)/total[h]*100
}' access.log | sort
2
Slow request detection and response time analysis
AWK
# Extended log with response time in last field ($NF)
# 10.0.1.5 - [01/May/2026:10:14:02] "GET /api/reports" 200 8420 1.847
# ── Requests over 1 second ────────────────────────────────
awk '$NF > 1.0 { printf "%.3fs %s %s\n", $NF, $7, $1 }' access.log \
| sort -rn
# ── Average response time per endpoint ────────────────────
awk '{
endpoint = $7
gsub(/\?.*/, "", endpoint) # strip query string
gsub(/\/[0-9]+/, "/:id", endpoint) # normalise IDs
sum[endpoint] += $NF
cnt[endpoint]++
if ($NF > max[endpoint]) max[endpoint] = $NF
}
END {
printf "%-40s %8s %8s %8s\n", "ENDPOINT", "AVG", "MAX", "COUNT"
printf "%-40s %8s %8s %8s\n", "--------", "---", "---", "-----"
for (e in sum)
printf "%-40s %7.3fs %7.3fs %8d\n",
e, sum[e]/cnt[e], max[e], cnt[e]
}' access.log | sort -k2 -rn | head -15
# ── P95 response time (sort + pick position) ──────────────
awk '{ print $NF }' access.log \
| sort -n \
| awk 'BEGIN{ pct=0.95 }
{ times[NR]=$1 }
END{ print "P95:", times[int(NR*pct)] "s" }'
3
Security — brute-force and anomaly detection
AWK
# ── SSH brute-force detection from /var/log/auth.log ──────
awk '/Failed password/ {
# "Failed password for root from 10.0.1.42 port 54321"
for (i=1; i<=NF; i++)
if ($i == "from") { ip=$(i+1); break }
fails[ip]++
}
END {
for (ip in fails)
if (fails[ip] > 10)
printf "BRUTE FORCE: %-18s %d failed attempts\n", ip, fails[ip]
}' /var/log/auth.log | sort -k3 -rn
# ── Detect login success after many failures (compromise) ─
awk '/Failed password/ {
match($0, /from ([0-9.]+)/, m)
fails[m[1]]++
}
/Accepted/ {
match($0, /from ([0-9.]+)/, m)
if (fails[m[1]] > 5)
print "SUSPICIOUS LOGIN after", fails[m[1]], "failures:", $0
}' /var/log/auth.log
# ── Rate limiting candidates from access log ──────────────
awk '{
# Extract minute window: [01/May/2026:10:14
match($4, /[0-9]+\/[A-Za-z]+\/[0-9]+:[0-9]+:[0-9]+/)
window = substr($4, RSTART, RLENGTH-3) # chop seconds
rate[window"~"$1]++
}
END {
for (k in rate) {
if (rate[k] > 60) { # more than 60 req/min
split(k, parts, "~")
printf "RATE: %-18s %d req/min window=%s\n",
parts[2], rate[k], parts[1]
}
}
}' access.log | sort -k2 -rn
4
Complete incident investigation workflow
BASH
#!/usr/bin/env bash
# incident_report.sh — Full AWK-powered incident snapshot
LOG="/var/log/nginx/access.log"
APP="/var/log/myapp/app.log"
SINCE="${1:-10:00}" # default: since 10:00 today
echo "=== Incident Report — since ${SINCE} ==="
echo ""
echo "── HTTP Error Summary ───────────────────────────────"
grep "${SINCE}" "${LOG}" | awk '$9~/^[45]/{code[$9]++}
END{for(c in code) printf " HTTP %s: %d\n",c,code[c]}' | sort
echo ""
echo "── Top Failing IPs ──────────────────────────────────"
grep "${SINCE}" "${LOG}" | awk '$9~/^[45]/{ip[$1]++}
END{for(i in ip) print ip[i],i}' | sort -rn | head -5 | \
awk '{ printf " %-5d %s\n", $1, $2 }'
echo ""
echo "── Slowest Endpoints ────────────────────────────────"
grep "${SINCE}" "${LOG}" | awk '$NF>0.5{
e=$7; gsub(/\?.*/,"",e); gsub(/\/[0-9]+/,"/:id",e)
sum[e]+=$NF; cnt[e]++}
END{for(e in sum) print sum[e]/cnt[e],e}' | \
sort -rn | head -5 | \
awk '{ printf " %6.3fs %s\n", $1, $2 }'
echo ""
echo "── Application Errors ───────────────────────────────"
grep "${SINCE}" "${APP}" | awk '/ERROR/{
msg=$0; gsub(/.*ERROR[[:space:]]*/,"",msg)
gsub(/[0-9a-f-]{36}/,"UUID",msg) # normalise UUIDs
errors[msg]++}
END{for(m in errors) print errors[m],m}' | \
sort -rn | head -5 | \
awk '{ printf " %-4d %s\n", $1, substr($0, index($0,$2)) }'
Terminal output
Key
Errors / threats
Slow / warning
Summary totals
vriddh@prod-01:~/scripts$./incident_report.sh "10:14"
=== Incident Report — since 10:14 ===
── HTTP Error Summary ───────────────────────────────
HTTP 500: 247
HTTP 429: 1842
HTTP 404: 38
── Top Failing IPs ──────────────────────────────────
1842 10.0.1.42
247 10.0.1.17
── Slowest Endpoints ────────────────────────────────
4.821s /api/reports
1.204s /api/users/:id
0.082s /api/health
█
✔ Log analysis with AWK — For Nginx logs, remember:
$1=IP, $4=timestamp, $7=URL, $9=status, $10=bytes, $NF=response time (if extended format). Always normalise endpoints by stripping query strings and replacing numeric IDs with :id before grouping. Use gsub(/UUID-pattern/, "UUID") to normalise error messages for grouping. Pipe to sort -rn | head for top-N without gawk's sort extension.