disable-headers-more.sh

disable-headers-more.sh

#!/usr/bin/env bash
#
# disable-headers-more.sh
# ------------------------
# Mitigates the nginx worker SIGSEGV caused by the out-of-tree
# headers-more dynamic module after the 1.24.0-2ubuntu7.10 security bump
# (module built against the old struct layout -> writes to a stale offset).
#
# What it does:
#   1. Verifies the host is Ubuntu and nginx is installed.
#   2. Backs up the entire /etc/nginx tree (timestamped tarball).
#   3. Disables the headers-more module (moves the modules-enabled symlink
#      into the backup, reversibly).
#   4. Comments out every more_set_headers / more_clear_headers directive
#      (per-file .hm-bak backups) so 'nginx -t' won't fail on unknown directive.
#   5. Tests the config. ONLY restarts if the test passes.
#   6. Verifies the service is active and not segfaulting after restart.
#
# Modes:
#   (default)     apply the mitigation
#   --dry-run     show what would change, touch nothing
#   --no-restart  apply + test but don't restart (for staged rollouts)
#   --rollback    re-enable the module and restore all commented directives
#   -h|--help     this help
#
# Exit codes: 0 ok | 1 usage/precondition | 2 not ubuntu | 3 nginx -t failed
#             4 still segfaulting after restart
#
set -euo pipefail

NGINX_DIR="/etc/nginx"
MODULES_ENABLED="${NGINX_DIR}/modules-enabled"
MODULE_LINK="${MODULES_ENABLED}/50-mod-http-headers-more-filter.conf"
MODULE_TARGET="/usr/share/nginx/modules-available/mod-http-headers-more-filter.conf"
BACKUP_ROOT="/var/backups/nginx-headers-more"
TS="$(date +%Y%m%d-%H%M%S)"
BACKUP_DIR="${BACKUP_ROOT}/${TS}"
LOG="/var/log/disable-headers-more-${TS}.log"

DRY_RUN=0
DO_RESTART=1
ROLLBACK=0

log()  { printf '%s [%s] %s\n' "$(date +%H:%M:%S)" "$1" "${2:-}" | tee -a "$LOG" >/dev/null 2>&1 || true; printf '[%s] %s\n' "$1" "${2:-}"; }
info() { log INFO  "$*"; }
warn() { log WARN  "$*"; }
err()  { log ERROR "$*"; }
run()  { # run a command, honoring --dry-run
  if [[ $DRY_RUN -eq 1 ]]; then printf '[DRY-RUN] %s\n' "$*"; else eval "$@"; fi
}

usage() { sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; }

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run)    DRY_RUN=1 ;;
    --no-restart) DO_RESTART=0 ;;
    --rollback)   ROLLBACK=1 ;;
    -h|--help)    usage 0 ;;
    *) err "unknown argument: $1"; usage 1 ;;
  esac
  shift
done

# --- preconditions ----------------------------------------------------------
[[ $EUID -eq 0 ]] || { err "must run as root"; exit 1; }

# Verify Ubuntu
if [[ -r /etc/os-release ]]; then
  . /etc/os-release
  if [[ "${ID:-}" != "ubuntu" ]]; then
    err "this host is '${ID:-unknown}', not ubuntu. Aborting."; exit 2
  fi
  info "OS: ${PRETTY_NAME:-Ubuntu} ($(uname -m)) on $(hostname)"
else
  err "/etc/os-release not found; cannot confirm Ubuntu. Aborting."; exit 2
fi

command -v nginx >/dev/null 2>&1 || { err "nginx not installed/in PATH. Aborting."; exit 1; }
[[ -d "$NGINX_DIR" ]] || { err "$NGINX_DIR not found. Aborting."; exit 1; }

mkdir -p "$BACKUP_DIR"
info "nginx version: $(nginx -v 2>&1 | sed 's/^nginx version: //')"
info "headers-more pkg: $(dpkg-query -W -f='${Version}' libnginx-mod-http-headers-more-filter 2>/dev/null || echo 'not installed')"

# --- rollback mode ----------------------------------------------------------
if [[ $ROLLBACK -eq 1 ]]; then
  info "ROLLBACK: restoring headers-more directives and module link"
  # restore every .hm-bak created by this script's sed step
  while IFS= read -r bak; do
    orig="${bak%.hm-bak}"
    info "restore ${orig}"
    run "mv -f '$bak' '$orig'"
  done < <(find "$NGINX_DIR" -type f -name '*.hm-bak' 2>/dev/null)
  # re-enable module link if the target exists and link is missing
  if [[ -e "$MODULE_TARGET" && ! -e "$MODULE_LINK" ]]; then
    info "re-enable module symlink"
    run "ln -s '$MODULE_TARGET' '$MODULE_LINK'"
  fi
  if run "nginx -t"; then
    [[ $DO_RESTART -eq 1 ]] && run "systemctl restart nginx" && info "nginx restarted"
    info "rollback complete"
    exit 0
  else
    err "nginx -t failed after rollback; inspect config before restarting"; exit 3
  fi
fi

# --- backup -----------------------------------------------------------------
info "backing up ${NGINX_DIR} -> ${BACKUP_DIR}/etc-nginx.tar.gz"
run "tar -C / -czf '${BACKUP_DIR}/etc-nginx.tar.gz' etc/nginx"

# warn if config includes anything outside /etc/nginx (script only sweeps /etc/nginx)
EXTERNAL_INCLUDES="$(grep -rhoE '^\s*include\s+\S+' "$NGINX_DIR" 2>/dev/null \
  | awk '{print $2}' | sed 's/;//' | grep -vE '^/etc/nginx|^[^/]' || true)"
if [[ -n "$EXTERNAL_INCLUDES" ]]; then
  warn "config includes paths OUTSIDE /etc/nginx — sweep them manually:"
  printf '%s\n' "$EXTERNAL_INCLUDES" | sed 's/^/         /'
fi

# --- disable module ---------------------------------------------------------
if [[ -e "$MODULE_LINK" || -L "$MODULE_LINK" ]]; then
  info "disabling module: $MODULE_LINK -> ${BACKUP_DIR}/"
  run "cp -a --no-dereference '$MODULE_LINK' '${BACKUP_DIR}/' 2>/dev/null || true"
  run "rm -f '$MODULE_LINK'"
else
  info "module link already absent (idempotent) — skipping"
fi

# --- comment directives -----------------------------------------------------
# anchored on a non-# start, so re-runs won't double-comment (idempotent)
mapfile -t HM_FILES < <(grep -rlE '^[[:space:]]*more_(set|clear)(_input)?_headers' "$NGINX_DIR" 2>/dev/null || true)
if [[ ${#HM_FILES[@]} -gt 0 ]]; then
  info "commenting more_* directives in ${#HM_FILES[@]} file(s)"
  for f in "${HM_FILES[@]}"; do
    info "  ${f}"
    run "sed -i.hm-bak -E 's/^([[:space:]]*)(more_(set|clear)(_input)?_headers)/\1# HM-DISABLED \2/' '$f'"
  done
else
  info "no active more_* directives found (idempotent) — skipping"
fi

# --- test -------------------------------------------------------------------
info "testing configuration"
if [[ $DRY_RUN -eq 1 ]]; then
  warn "dry-run: skipping 'nginx -t' and restart"
  info "dry-run complete; backup at ${BACKUP_DIR}"
  exit 0
fi

if ! nginx -t 2>&1 | tee -a "$LOG"; then
  err "nginx -t FAILED — NOT restarting."
  err "Likely an unswept more_* directive (split line, or a path listed above)."
  err "Roll back with:  $0 --rollback"
  exit 3
fi
info "nginx -t passed"

# --- restart + verify -------------------------------------------------------
if [[ $DO_RESTART -eq 0 ]]; then
  info "--no-restart set; config staged. Restart manually when ready."
  exit 0
fi

SINCE="$(date '+%Y-%m-%d %H:%M:%S')"
info "restarting nginx"
systemctl restart nginx
sleep 4

if ! systemctl is-active --quiet nginx; then
  err "nginx is not active after restart — check 'journalctl -u nginx'"
  err "Roll back with:  $0 --rollback"
  exit 4
fi

NEW_SEGV="$(journalctl -k --since "$SINCE" 2>/dev/null | grep -ic 'nginx.*segfault' || true)"
if [[ "${NEW_SEGV:-0}" -gt 0 ]]; then
  err "STILL SEGFAULTING after restart (${NEW_SEGV} events) — headers-more was not the sole cause."
  err "This points to a genuine 7.10 core regression; pull 7.9 from Launchpad:"
  err "  apt-get install ubuntu-dev-tools && pull-lp-debs nginx 1.24.0-2ubuntu7.9"
  exit 4
fi

CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://localhost/ || echo 000)"
info "local HTTP check: ${CODE}"
info "SUCCESS: nginx active, no new segfaults. Backup: ${BACKUP_DIR}"
info "Next: rebuild headers-more against the running nginx, then '$0 --rollback' to restore directives."
exit 0