#!/bin/bash
# =============================================================================
# Panelica Server Panel - Installation Script
# =============================================================================
# Supported: Ubuntu 22.04+, Debian 12+, AlmaLinux 9, Rocky 9
# Usage:
#   curl -sSL https://latest.panelica.com/install.sh | bash     (download + install)
#   bash install.sh --local /tmp/panelica-v1.0.0.tar.gz         (local tarball)
# =============================================================================

set -uo pipefail

# --- Configuration ---
INSTALL_DIR="/opt/panelica"
PANELICA_VERSION=""
MIN_RAM_MB="${PANELICA_MIN_RAM_MB:-1024}"
MIN_DISK_GB="${PANELICA_MIN_DISK_GB:-15}"
DOWNLOAD_URL="https://latest.panelica.com/panelica-latest.tar.gz"
CHECKSUM_URL="https://latest.panelica.com/panelica-latest.sha256"
LOCAL_TARBALL=""
LOG_FILE="/tmp/panelica-install.log"
CENTRAL_URL="https://panelica.com"

# --- Parse arguments ---
while [[ $# -gt 0 ]]; do
    case $1 in
        --local)
            LOCAL_TARBALL="$2"
            shift 2
            ;;
        --version)
            PANELICA_VERSION="$2"
            shift 2
            ;;
        --central-url)
            CENTRAL_URL="$2"
            shift 2
            ;;
        *)
            shift
            ;;
    esac
done

# Auto-detect version from tarball filename (panelica-v1.0.7.tar.gz → 1.0.7)
if [ -n "${LOCAL_TARBALL}" ] && [ -z "${PANELICA_VERSION}" ]; then
    DETECTED_VER=$(basename "${LOCAL_TARBALL}" | sed -n 's/panelica-v\([0-9.]*\)\.tar\.gz/\1/p')
    if [ -n "${DETECTED_VER}" ]; then
        PANELICA_VERSION="${DETECTED_VER}"
    fi
fi

# --- Colors & Symbols ---
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
BLUE='\033[0;34m'
MAGENTA='\033[0;35m'
BOLD='\033[1m'
DIM='\033[2m'
NC='\033[0m'

CHECKMARK="${GREEN}✓${NC}"
CROSSMARK="${RED}✗${NC}"
ARROW="${CYAN}→${NC}"
BULLET="${DIM}•${NC}"

# --- Logging functions ---
log_info()    { echo -e "  ${CHECKMARK} $1" | tee -a "${LOG_FILE}"; }
log_warn()    { echo -e "  ${YELLOW}⚠${NC} $1" | tee -a "${LOG_FILE}"; }
log_error()   { echo -e "  ${CROSSMARK} $1" | tee -a "${LOG_FILE}"; }
log_detail()  { echo -e "    ${BULLET} $1" | tee -a "${LOG_FILE}"; }
log_step() {
    local step_num=$1
    local total=$2
    local title=$3
    local pct=$(( step_num * 100 / total ))
    local filled=$(( pct / 5 ))
    local empty=$(( 20 - filled ))
    local bar=$(printf '%0.s█' $(seq 1 $filled 2>/dev/null) 2>/dev/null)$(printf '%0.s░' $(seq 1 $empty 2>/dev/null) 2>/dev/null)

    echo "" | tee -a "${LOG_FILE}"
    echo -e "  ${CYAN}[${step_num}/${total}]${NC} ${BOLD}${title}${NC}" | tee -a "${LOG_FILE}"
    echo -e "  ${DIM}${bar} ${pct}%${NC}" | tee -a "${LOG_FILE}"
    echo "" | tee -a "${LOG_FILE}"
}

TOTAL_STEPS=28

# Save direct terminal fd BEFORE tee redirect (for spinner animation)
exec 3>&1

# Redirect all output to log file too
exec > >(tee -a "${LOG_FILE}") 2>&1

# --- Spinner for long-running commands ---
# Usage: run_with_spinner "message" command arg1 arg2 ...
# Writes spinner animation directly to terminal (fd 3), bypassing tee
run_with_spinner() {
    local msg="$1"
    shift
    local spin_chars='⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'

    # Run command in background, output to log only
    "$@" >> "${LOG_FILE}" 2>&1 &
    local pid=$!

    # Show spinner on terminal while command runs
    local i=0
    while kill -0 "$pid" 2>/dev/null; do
        local c="${spin_chars:$((i % ${#spin_chars})):1}"
        printf "\r    \033[0;36m%s\033[0m %s" "$c" "$msg" >&3
        i=$((i + 1))
        sleep 0.12
    done

    wait "$pid"
    local rc=$?

    # Clear spinner line and log result
    printf "\r%-70s\r" "" >&3
    if [ $rc -eq 0 ]; then
        echo -e "    ${BULLET} ${msg} ${GREEN}done${NC}"
    else
        echo -e "    ${BULLET} ${msg} ${RED}failed${NC}"
    fi

    return $rc
}

# --- ASCII Art Banner ---
echo ""
echo -e "${CYAN}"
cat << 'BANNER'
    ██████╗  █████╗ ███╗   ██╗███████╗██╗     ██╗ ██████╗ █████╗
    ██╔══██╗██╔══██╗████╗  ██║██╔════╝██║     ██║██╔════╝██╔══██╗
    ██████╔╝███████║██╔██╗ ██║█████╗  ██║     ██║██║     ███████║
    ██╔═══╝ ██╔══██║██║╚██╗██║██╔══╝  ██║     ██║██║     ██╔══██║
    ██║     ██║  ██║██║ ╚████║███████╗███████╗██║╚██████╗██║  ██║
    ╚═╝     ╚═╝  ╚═╝╚═╝  ╚═══╝╚══════╝╚══════╝╚═╝ ╚═════╝╚═╝  ╚═╝
BANNER
echo -e "${NC}"
if [ -n "${PANELICA_VERSION}" ]; then
    _VER_DISPLAY="v${PANELICA_VERSION}"
else
    _VER_DISPLAY="(latest)"
fi
echo -e "    ${BOLD}Server Management Panel${NC}  ${DIM}${_VER_DISPLAY}${NC}"
echo -e "    ${DIM}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo ""
echo -e "    ${BULLET} Modern hosting control panel with enterprise security"
echo -e "    ${BULLET} 5-layer isolation: Cgroups v2, Namespaces, Chroot, PHP-FPM, Unix"
echo -e "    ${BULLET} 20+ integrated services, auto-configured and ready to use"
echo ""
echo -e "    ${DIM}Installation log: ${LOG_FILE}${NC}"
echo -e "    ${DIM}Started at: $(date '+%Y-%m-%d %H:%M:%S %Z')${NC}"
echo ""

# =============================================================================
# STEP 1: System Requirements Check
# =============================================================================
log_step 1 ${TOTAL_STEPS} "Checking System Requirements"

# Root check
if [ "$(id -u)" -ne 0 ]; then
    log_error "This script must be run as root!"
    exit 1
fi

# OS check
if [ ! -f /etc/os-release ]; then
    log_error "Unable to detect operating system!"
    exit 1
fi

source /etc/os-release
OS_ID="${ID}"
OS_VER="${VERSION_ID}"

# OS family — Panelica supports Debian-family (Ubuntu/Debian) and RHEL-family
# (AlmaLinux/Rocky 9). RHEL 9 family defaults to cgroup v2 (required by Panelica
# isolation); RHEL/Alma/Rocky 8 default to cgroup v1 and are intentionally unsupported.
case "${OS_ID}" in
    ubuntu|debian)               OS_FAMILY="debian" ;;
    almalinux|rocky|rhel|centos) OS_FAMILY="rhel" ;;
    *)                           OS_FAMILY="" ;;
esac
if [ -z "${OS_FAMILY}" ]; then
    log_error "Unsupported OS: ${OS_ID}. Supported: Ubuntu 22.04+, Debian 12+, AlmaLinux 9, Rocky 9"
    exit 1
fi

OS_MAJOR=$(echo "${VERSION_ID}" | cut -d. -f1)
if [ "${OS_ID}" = "ubuntu" ] && [ "${OS_MAJOR}" -lt 22 ]; then
    log_error "Ubuntu 22.04 or later required! Detected: ${VERSION_ID}"
    exit 1
elif [ "${OS_ID}" = "debian" ] && [ "${OS_MAJOR}" -lt 12 ]; then
    log_error "Debian 12 or later required! Detected: ${VERSION_ID}"
    exit 1
elif [ "${OS_FAMILY}" = "rhel" ] && [ "${OS_MAJOR}" -lt 9 ]; then
    log_error "AlmaLinux/Rocky 9 or later required (cgroup v2)! Detected: ${VERSION_ID}"
    exit 1
fi
log_info "Operating system: ${PRETTY_NAME} (family: ${OS_FAMILY})"

# GLIBC compatibility check — bundled binaries are built against GLIBC 2.34
# (AlmaLinux 9 baseline). glibc is backward-compatible, so 2.34 binaries run on
# 2.35+ (Ubuntu 22.04+, Debian 12+) too. Catches older OSes (RHEL 8, Debian 11,
# Ubuntu 20.04) that pass the OS check but lack required symbols.
REQUIRED_GLIBC="2.34"
CURRENT_GLIBC=$(ldd --version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+$')
if [ -z "${CURRENT_GLIBC}" ]; then
    log_error "Cannot detect GLIBC version (ldd output unexpected)"
    exit 1
fi
# Distro-independent version compare (no dpkg dependency — works on RHEL too)
if [ "$(printf '%s\n%s\n' "${CURRENT_GLIBC}" "${REQUIRED_GLIBC}" | sort -V | head -1)" != "${REQUIRED_GLIBC}" ]; then
    log_error "GLIBC ${CURRENT_GLIBC} is too old. Panelica requires GLIBC ${REQUIRED_GLIBC} or newer."
    log_error "This usually means an older OS (e.g., RHEL 8, Debian 11, Ubuntu 20.04) — please upgrade."
    exit 1
fi
log_info "GLIBC version: ${CURRENT_GLIBC} (>= ${REQUIRED_GLIBC} required)"

# RAM check — read /proc/meminfo directly so we do not depend on `free`
# (procps is absent on minimal Debian images; `free` then errors out).
TOTAL_RAM_MB=$(awk '/^MemTotal:/{printf "%d", $2/1024}' /proc/meminfo 2>/dev/null)
[ -z "${TOTAL_RAM_MB}" ] && TOTAL_RAM_MB=$(free -m 2>/dev/null | awk '/^Mem:/{print $2}')
[ -z "${TOTAL_RAM_MB}" ] && TOTAL_RAM_MB=0
if [ "${TOTAL_RAM_MB}" -lt "${MIN_RAM_MB}" ]; then
    log_error "Minimum ${MIN_RAM_MB}MB RAM required! Available: ${TOTAL_RAM_MB}MB"
    exit 1
fi
log_info "Memory: ${TOTAL_RAM_MB}MB available"

# Disk check
AVAIL_DISK_GB=$(df -BG /opt | awk 'NR==2{print $4}' | tr -d 'G')
if [ "${AVAIL_DISK_GB}" -lt "${MIN_DISK_GB}" ]; then
    log_error "Minimum ${MIN_DISK_GB}GB free disk space required! Available: ${AVAIL_DISK_GB}GB"
    exit 1
fi
log_info "Disk space: ${AVAIL_DISK_GB}GB free"

# CPU info
CPU_CORES=$(nproc 2>/dev/null || echo "?")
log_info "CPU cores: ${CPU_CORES}"

# Check if already installed (allow re-run after failed install)
if [ -d "${INSTALL_DIR}/bin" ] && systemctl is-active --quiet panelica-backend 2>/dev/null; then
    log_error "Panelica is already installed and running at ${INSTALL_DIR}!"
    log_error "To reinstall, first run: /opt/panelica/bin/pn-service stop all && rm -rf /opt/panelica"
    exit 1
fi
# Clean up partial install if exists
if [ -d "${INSTALL_DIR}" ] && [ ! -f "${INSTALL_DIR}/bin/panelica-server" ]; then
    log_warn "Removing partial installation from previous attempt..."
    rm -rf "${INSTALL_DIR}"
fi

# Auto-detect server IP
SERVER_IP=$(ip -4 route get 8.8.8.8 2>/dev/null | awk '{print $7; exit}')
if [ -z "${SERVER_IP}" ]; then
    SERVER_IP=$(hostname -I | awk '{print $1}')
fi
log_info "Server IP: ${SERVER_IP}"

# Hostname (prefer FQDN, fallback to IP if hostname is useless)
SERVER_HOSTNAME=$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo "")
# If hostname is empty, localhost, or a bare name without dots, use IP instead
if [ -z "${SERVER_HOSTNAME}" ] || [ "${SERVER_HOSTNAME}" = "localhost" ] || ! echo "${SERVER_HOSTNAME}" | grep -q '\.'; then
    SERVER_HOSTNAME="${SERVER_IP}"
fi
# Normalize to lowercase — prevents PTR/SPF/DKIM/SSL mismatches
SERVER_HOSTNAME=$(echo "${SERVER_HOSTNAME}" | tr 'A-Z' 'a-z')
log_info "Hostname: ${SERVER_HOSTNAME}"

# Normalize system hostname to lowercase (prevents /etc/hostname mixed case issues)
# hostnamectl updates both /etc/hostname and the running kernel hostname
if [ "${SERVER_HOSTNAME}" != "${SERVER_IP}" ]; then
    CURRENT_SYS_HOSTNAME=$(cat /etc/hostname 2>/dev/null | tr -d '\n')
    CURRENT_SYS_LOWER=$(echo "${CURRENT_SYS_HOSTNAME}" | tr 'A-Z' 'a-z')
    if [ "${CURRENT_SYS_HOSTNAME}" != "${CURRENT_SYS_LOWER}" ] || [ "${CURRENT_SYS_LOWER}" != "${SERVER_HOSTNAME}" ]; then
        hostnamectl set-hostname "${SERVER_HOSTNAME}" 2>/dev/null || echo "${SERVER_HOSTNAME}" > /etc/hostname
        log_detail "System hostname normalized to: ${SERVER_HOSTNAME}"
    fi
    # Ensure /etc/hosts has the lowercase hostname
    if grep -q "^127\.0\.1\.1" /etc/hosts 2>/dev/null; then
        sed -i "s/^127\.0\.1\.1.*/127.0.1.1\t${SERVER_HOSTNAME}/" /etc/hosts
    else
        sed -i "/^127\.0\.0\.1/a 127.0.1.1\t${SERVER_HOSTNAME}" /etc/hosts
    fi

    # Fix server public IP in /etc/hosts (critical for mail deliverability)
    # Hosting providers (Hetzner, OVH, etc.) leave generic hostnames like "Ubuntu-2404-noble-amd64-base"
    # which appear in Received headers and trigger spam filters
    _MAIL_HOSTNAME="mail.$(echo "${SERVER_HOSTNAME}" | sed 's/^[^.]*\.//')"
    if grep -q "^${SERVER_IP}" /etc/hosts 2>/dev/null; then
        sed -i "s/^${SERVER_IP}.*/${SERVER_IP} ${SERVER_HOSTNAME} ${_MAIL_HOSTNAME}/" /etc/hosts
    else
        echo "${SERVER_IP} ${SERVER_HOSTNAME} ${_MAIL_HOSTNAME}" >> /etc/hosts
    fi
    log_detail "Fixed /etc/hosts: ${SERVER_IP} -> ${SERVER_HOSTNAME} ${_MAIL_HOSTNAME}"

    # Also fix any IPv6 lines with generic hostnames (Hetzner sets these)
    sed -i -E "s/^(2[0-9a-f:]+)\s+(Ubuntu-|Debian-|Hetzner-|OVH-|vps)[^ ]*/\1 ${SERVER_HOSTNAME}/i" /etc/hosts 2>/dev/null || true
fi

# =============================================================================
# STEP 2: Install System Dependencies
# =============================================================================
log_step 2 ${TOTAL_STEPS} "Installing System Dependencies"

# Debian-family (apt) and RHEL-family (dnf) take separate package-install paths.
# The Debian path below is byte-for-byte unchanged; the RHEL path is the elif.
if [ "${OS_FAMILY}" = "debian" ]; then

export DEBIAN_FRONTEND=noninteractive
# Prevent needrestart from triggering interactive prompts or service restarts
# during apt-get operations — avoids deadlocks with VPS first-boot scripts
# (e.g. debian-based-post-install.service on Vultr/Hetzner/DO Ubuntu 24.04 images)
# and prevents accidental sshd restart that would drop our install session
export NEEDRESTART_SUSPEND=1
export NEEDRESTART_MODE=l

# Ensure DNS works (systemd-resolved may have been disabled by a previous failed install)
if ! host google.com >/dev/null 2>&1 && ! ping -c1 -W2 8.8.8.8 >/dev/null 2>&1; then
    log_detail "DNS not working, setting static resolvers..."
fi
# Always ensure /etc/resolv.conf has working nameservers (needed after systemd-resolved disabled)
if ! grep -q "^nameserver" /etc/resolv.conf 2>/dev/null || systemctl is-active --quiet systemd-resolved 2>/dev/null; then
    # Will set proper resolv.conf later when we disable systemd-resolved
    true
else
    # systemd-resolved already disabled (re-run) but resolv.conf might be a dead symlink
    if [ -L /etc/resolv.conf ] && [ ! -e /etc/resolv.conf ]; then
        rm -f /etc/resolv.conf
        printf "nameserver 8.8.8.8\nnameserver 1.1.1.1\n" > /etc/resolv.conf
        log_detail "Restored DNS resolvers (broken symlink fixed)"
    fi
fi

# Kill unattended-upgrades and wait for APT lock (with spinner)
_release_apt_lock() {
    systemctl stop unattended-upgrades 2>/dev/null || true
    systemctl disable unattended-upgrades 2>/dev/null || true
    systemctl kill unattended-upgrades 2>/dev/null || true
    pkill -9 -f unattended-upgrade 2>/dev/null || true
    pkill -9 -f "apt.systemd.daily" 2>/dev/null || true
    # VPS provider first-boot scripts (Vultr/Hetzner/DO Ubuntu 24.04+):
    # one-off services that run `apt update && apt upgrade` on first boot.
    # If left running, they deadlock with our apt-get install via needrestart.
    for svc in debian-based-post-install ubuntu-cloud-init-post first-boot; do
        systemctl stop "${svc}.service" 2>/dev/null || true
        systemctl disable "${svc}.service" 2>/dev/null || true
        systemctl kill "${svc}.service" 2>/dev/null || true
    done
    pkill -9 -f debian-based-post-install 2>/dev/null || true
    sleep 2
    for i in $(seq 1 15); do
        fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || break
        [ $i -eq 15 ] && { fuser -k /var/lib/dpkg/lock-frontend 2>/dev/null || true; sleep 2; }
        sleep 2
    done
    dpkg --configure -a 2>/dev/null || true
}
run_with_spinner "Releasing APT lock (stopping background updaters)..." _release_apt_lock

# Pre-empt grub-pc / kernel partial-configure traps left behind by cloud-init
# or first-boot auto-upgrades on freshly-imaged Debian/Ubuntu VMs. grub-pc's
# postinst asks "which disk?" with debconf priority=critical, so even with
# DEBIAN_FRONTEND=noninteractive it fails for lack of a default. Pre-seed the
# install_devices with the actual root disk, hold grub-pc + grub-common so apt
# won't touch them, and clear any leftover partial-configure state.
_settle_apt_grub() {
    local root_dev=$(findmnt -no SOURCE / 2>/dev/null | sed 's/[0-9]*$//' || echo "/dev/sda")
    echo "grub-pc grub-pc/install_devices multiselect ${root_dev}" | debconf-set-selections 2>/dev/null || true
    echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections 2>/dev/null || true
    apt-mark hold grub-pc grub-common 2>/dev/null || true
    dpkg --configure -a 2>/dev/null || true
}
run_with_spinner "Settling package state (cloud-init cleanup)..." _settle_apt_grub

run_with_spinner "Updating package lists..." apt-get update -qq

# Detect OS for package name differences
OS_VER="${VERSION_ID}"
OS_ID="${ID}"
log_detail "Detected ${OS_ID} ${OS_VER}"

# t64/soname transition: pick first available candidate (Ubuntu 26.04+ removes legacy names)
# 22.04: legacy names | 24.04: both available | 26.04+: only t64 names
_pick_pkg() {
    for c in "$@"; do
        apt-cache show "$c" >/dev/null 2>&1 && { echo "$c"; return 0; }
    done
    echo "$1"
}
LIB_SSL=$(_pick_pkg libssl3t64 libssl3)
LIB_XML=$(_pick_pkg libxml2-16 libxml2)
LIB_CURL=$(_pick_pkg libcurl4t64 libcurl4)
LIB_UV=$(_pick_pkg libuv1t64 libuv1)

# Common packages (exist on Ubuntu 22.04+, Debian 12+)
# tzdata: required by PostgreSQL initdb (/usr/share/zoneinfo) — absent on
#   minimal Ubuntu 22.04 base images.
# procps: provides `free`/`ps`/`pgrep` — absent on minimal Debian images.
COMMON_PKGS="tar gzip curl wget openssl jq rsync sshpass lsb-release sudo
    tzdata procps zstd
    libnuma1 liburing2 libcap2
    ${LIB_SSL} ${LIB_XML} ${LIB_CURL}
    libonig5 libsodium23
    libavif-bin libgd3
    libenchant-2-2 libxslt1.1
    libmaxminddb0 ${LIB_UV} liblua5.3-0 libyajl2
    libgomp1
    dns-root-data
    nftables
    apparmor apparmor-utils
    fail2ban
    clamav clamav-daemon clamav-freshclam
    opendkim opendkim-tools
    logrotate
    acl
    redis-server"

# Packages with different names across distros — detect dynamically
_try_pkg() { apt-cache show "$1" >/dev/null 2>&1 && COMMON_PKGS="${COMMON_PKGS} $1" && return 0; return 1; }

# libpcre3 (Ubuntu) vs libpcre2 (Debian 13+)
_try_pkg libpcre3 || true

# IMAP C-client: libc-client2007e (Ubuntu/Debian 12) — not available on Debian 13
_try_pkg libc-client2007e || log_detail "libc-client2007e not available (Debian 13+ — PHP IMAP uses bundled)"

# ODBC: unixodbc provides libodbc.so.2 (needed by PHP odbc/pdo_odbc extensions)
_try_pkg unixodbc || true

# HTML Tidy: libtidy5deb1 (Ubuntu/Debian 12) vs libtidy58 (Debian 13)
_try_pkg libtidy5deb1 || _try_pkg libtidy58 || true

# libzip: libzip4 (Ubuntu/Debian 12) vs libzip5 (Debian 13)
_try_pkg libzip4 || _try_pkg libzip5 || true

# Python crypt module: removed in Python 3.13 (PEP 594) → required by FTP/SSH password hashing
# Ubuntu 26.04+ ships 'python3-crypt-r' as drop-in replacement (makes 'import crypt' work)
_try_pkg python3-crypt-r || true

# OS-specific packages (library names changed in time64 ABI transition)
# Detect libmagickwand package name dynamically (major version + suffix changes across releases)
MAGICKWAND_PKG=$(apt-cache search --names-only '^libmagickwand-[67]\.q16-[0-9]+' 2>/dev/null | grep -v '\-dev' | grep -v 'hdri' | head -1 | awk '{print $1}')
[ -z "$MAGICKWAND_PKG" ] && MAGICKWAND_PKG="libmagickwand-6.q16-6"
log_detail "ImageMagick package: ${MAGICKWAND_PKG}"

# time64 ABI: Ubuntu 24.04+ or Debian 13+ use t64 library names
# Debian 12: non-t64, libncurses6  |  Ubuntu 22.04: non-t64, libncurses5
# Debian 13: t64, libncurses6      |  Ubuntu 24.04: t64, libncurses6
if [[ "${OS_ID}" = "debian" && "${OS_MAJOR}" -ge 13 ]] || [[ "${OS_ID}" = "ubuntu" && "${OS_MAJOR}" -ge 24 ]]; then
    OS_PKGS="libaio1t64 libncurses6 libsnmp40t64 libldap2 libnsl2 ${MAGICKWAND_PKG}"
elif [[ "${OS_ID}" = "debian" ]]; then
    OS_PKGS="libaio1 libncurses6 libsnmp40 libldap-2.5-0 ${MAGICKWAND_PKG}"
else
    OS_PKGS="libaio1 libncurses5 libsnmp40 libldap-2.5-0 ${MAGICKWAND_PKG}"
fi

# Resilient package install — a fresh machine/container can hit transient
# apt-index or repo errors on the first try ("Unable to locate package").
# Retry with a fresh `apt-get update` in between instead of aborting on the
# first failure (parity with the RHEL branch's retry resilience).
_install_deb_pkgs() {
    apt-get install -y --fix-missing ${COMMON_PKGS} ${OS_PKGS} && return 0
    # 1st retry — refresh package lists (handles stale/empty index)
    apt-get update -qq 2>/dev/null || true
    apt-get install -y --fix-missing ${COMMON_PKGS} ${OS_PKGS} && return 0
    # 2nd retry — repair dpkg state + reconcile broken deps, then re-try.
    # `apt-get install -f` finishes any half-configured packages that pre-seed
    # alone cannot resolve (e.g. cloud-init left a kernel upgrade mid-flight).
    dpkg --configure -a 2>/dev/null || true
    apt-get install -f -y 2>/dev/null || true
    apt-get update -qq 2>/dev/null || true
    apt-get install -y --fix-missing ${COMMON_PKGS} ${OS_PKGS}
}
run_with_spinner "Installing required packages..." _install_deb_pkgs
if [ $? -ne 0 ]; then
    log_error "Failed to install required packages! Check network/DNS and try again."
    log_error "Log: ${LOG_FILE}"
    exit 1
fi

elif [ "${OS_FAMILY}" = "rhel" ]; then
    # ===== RHEL-family (AlmaLinux / Rocky 9) — dnf =====
    log_detail "Detected ${OS_ID} ${OS_VER} (RHEL family)"

    # DNS sanity — repair a broken resolv.conf symlink (parity with Debian path)
    if [ -L /etc/resolv.conf ] && [ ! -e /etc/resolv.conf ]; then
        rm -f /etc/resolv.conf
        printf "nameserver 8.8.8.8\nnameserver 1.1.1.1\n" > /etc/resolv.conf
        log_detail "Restored DNS resolvers (broken symlink fixed)"
    fi

    # Wait for any provider-side first-boot dnf / dnf-automatic / packagekit
    # to finish before we touch dnf ourselves. Hetzner/Vultr/DO AlmaLinux/Rocky
    # cloud images run `dnf update` on first boot — racing them triggers
    # "Database file is locked" errors that look like Panelica install bugs.
    _settle_dnf_state() {
        local own_pid=$$
        for i in $(seq 1 60); do
            local busy=$(pgrep -f "dnf|yum|packagekit|dnf-automatic" 2>/dev/null | grep -v "^${own_pid}$" | head -1)
            [ -z "$busy" ] && break
            sleep 5
        done
        # Clear stale metadata and finalize any incomplete transaction
        dnf clean expire-cache >/dev/null 2>&1 || true
        rpm --rebuilddb >/dev/null 2>&1 || true
    }
    run_with_spinner "Settling package manager (waiting for first-boot updates)..." _settle_dnf_state

    # EPEL + CRB — oniguruma, libsodium, fail2ban, clamd, opendkim, libmaxminddb,
    # and other runtime libraries live in these repositories.
    # NOTE: yajl is shipped in lib/compat/ (EPEL 10 dropped it); redis is bundled.
    _rhel_repos() {
        dnf -y install dnf-plugins-core epel-release
        dnf config-manager --set-enabled crb 2>/dev/null \
            || dnf config-manager --set-enabled powertools 2>/dev/null || true
        dnf -y makecache
    }
    run_with_spinner "Enabling EPEL/CRB repositories..." _rhel_repos

    # Runtime packages — RHEL/EPEL equivalents of the Debian COMMON_PKGS+OS_PKGS set.
    # NOT installed on RHEL: apparmor* (RHEL uses SELinux), dns-root-data (bundled
    # BIND ships its own root hints).
    # NOTE: curl/libcurl are intentionally omitted — AlmaLinux/Rocky 9 ship
    # curl-minimal/libcurl-minimal in the base image (they provide the curl
    # command and libcurl.so.4); installing full 'curl' conflicts with them.
    # libxcrypt-compat provides the legacy libcrypt.so.1 ABI — bundled apache,
    # proftpd and nginx link against it (RHEL 9 base ships only libcrypt.so.2).
    RHEL_PKGS="tar gzip wget openssl jq rsync sshpass sudo hostname iproute
        tzdata procps-ng zstd
        numactl-libs liburing libnsl2 libxcrypt-compat libtirpc
        openssl-libs libxml2 pcre2
        oniguruma libsodium
        libavif gd
        enchant2 libxslt
        libmaxminddb libuv lua-libs
        libgomp
        nftables logrotate acl
        fail2ban
        clamav clamd clamav-update
        opendkim opendkim-tools
        libaio ncurses-libs net-snmp-libs openldap
        unixODBC libtidy libzip
        ImageMagick-libs
        python3 python3-pip"

    # Bulk install; on failure retry package-by-package so a single missing
    # optional package never aborts the whole install.
    if ! run_with_spinner "Installing required packages..." dnf -y install ${RHEL_PKGS}; then
        log_warn "Bulk package install had errors — retrying package-by-package..."
        _rhel_missing=0
        for _p in ${RHEL_PKGS}; do
            dnf -y install "${_p}" >> "${LOG_FILE}" 2>&1 \
                || { log_detail "package unavailable: ${_p}"; _rhel_missing=$((_rhel_missing+1)); }
        done
        if [ "${_rhel_missing}" -gt 8 ]; then
            log_error "Too many packages failed to install (${_rhel_missing}) — check network/repos"
            exit 1
        fi
    fi
fi

log_info "All system packages installed successfully"

# =============================================================================
# STEP 3: Disable Conflicting Services
# =============================================================================
log_step 3 ${TOTAL_STEPS} "Reserving System Resources"

# unattended-upgrades already disabled in Step 2 (APT lock prevention)

# systemd-resolved - conflicts with BIND on port 53
# CRITICAL: Set up static DNS BEFORE disabling resolved to avoid DNS blackout
rm -f /etc/resolv.conf
printf "nameserver 8.8.8.8\nnameserver 1.1.1.1\nnameserver 8.8.4.4\n" > /etc/resolv.conf
systemctl stop systemd-resolved 2>/dev/null || true
systemctl disable systemd-resolved 2>/dev/null || true
log_detail "Reserved port 53 for Panelica DNS (BIND)"

# ClamAV AppArmor profiles - block access to /opt/panelica/ (Debian-family only;
# RHEL has no AppArmor — it uses SELinux, handled separately). Panelica ships
# its own clamav under services/clamav/, so when the system clamav package
# wasn't pulled in (most installs) the corresponding profiles never existed
# either — aa-disable then emits "ERROR: Profile doesn't exist" which is
# harmless but pollutes the install log. We check whether the profile is
# actually loaded before touching it, and pipe stderr to /dev/null as a
# belt-and-suspenders cover.
if [ "${OS_FAMILY}" = "debian" ] && command -v aa-status >/dev/null 2>&1; then
    AA_LOADED=$(aa-status --enabled 2>/dev/null && aa-status 2>/dev/null || true)
    for _aa_bin in /usr/bin/freshclam /usr/sbin/clamd; do
        if echo "${AA_LOADED}" | grep -q "$(basename "${_aa_bin}")"; then
            aa-disable "${_aa_bin}" >> "${LOG_FILE}" 2>/dev/null || true
        fi
    done
    log_detail "Disabled ClamAV AppArmor profiles (allows /opt/panelica access)"
fi

# Stop system clamav services (we use our own managed instances)
systemctl stop clamav-freshclam 2>/dev/null || true
systemctl stop clamav-daemon 2>/dev/null || true
systemctl disable clamav-freshclam 2>/dev/null || true
systemctl disable clamav-daemon 2>/dev/null || true
log_detail "Using Panelica's bundled antivirus (optimized for the panel)"

# SELinux — RHEL ships it enforcing, which blocks Panelica writes under
# /opt/panelica, socket binds, chroot and namespace operations. We disable
# it entirely on first install so customers don't see denial logs in
# journalctl/audit.log for opendkim, bandwidth.log, namespaces, etc.
# `setenforce 0` flips runtime to permissive immediately (disabled requires
# a reboot to take full effect, but config persistence handles that).
# A proper targeted SELinux policy is future work — until then disabling
# is the supported configuration for Panelica on RHEL-family hosts.
if [ "${OS_FAMILY}" = "rhel" ]; then
    command -v setenforce >/dev/null 2>&1 && setenforce 0 2>/dev/null || true
    if [ -f /etc/selinux/config ]; then
        sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
    fi
    # During this install session SELinux is still loaded in permissive
    # mode (full disable kicks in on next boot). Permissive doesn't block
    # anything, but it does fan-out "SELinux is preventing ..." messages
    # via setroubleshootd to journalctl/audit.log — which then look like
    # real errors to anyone tailing the journal post-install. Mask the
    # daemon so its noisy AVCs stop polluting logs. The kernel audit
    # subsystem stays up for unrelated kernel needs; only the user-facing
    # denial reporter is silenced.
    systemctl stop setroubleshootd 2>/dev/null || true
    systemctl disable setroubleshootd 2>/dev/null || true
    systemctl mask setroubleshootd 2>/dev/null || true
    log_detail "SELinux disabled for Panelica (runtime set permissive; config disabled takes effect after reboot; setroubleshootd masked to silence transient denials)"
fi

log_info "System resources reserved for Panelica"

# =============================================================================
# STEP 4: Create System Users & Groups
# =============================================================================
log_step 4 ${TOTAL_STEPS} "Creating System Users & Groups"

create_user() {
    local username=$1
    local groupname=${2:-$1}
    local shell=${3:-/sbin/nologin}
    local home=${4:-}

    if ! getent group "${groupname}" >/dev/null 2>&1; then
        groupadd -r "${groupname}"
    fi
    if ! id "${username}" >/dev/null 2>&1; then
        if [ -n "${home}" ]; then
            useradd -r -g "${groupname}" -s "${shell}" -d "${home}" "${username}"
        else
            useradd -r -g "${groupname}" -s "${shell}" "${username}"
        fi
        log_detail "Created user ${username}:${groupname}"
    else
        log_detail "User ${username} already exists"
    fi
}

create_user postgres postgres /sbin/nologin "${INSTALL_DIR}/var/db/postgresql"
create_user mysql mysql /sbin/nologin "${INSTALL_DIR}/var/db/mysql"
create_user redis redis
create_user bind bind
create_user vmail vmail /sbin/nologin /var/vmail
# Ensure vmail home directory exists with correct ownership
mkdir -p /var/vmail
chown vmail:vmail /var/vmail
chmod 770 /var/vmail
log_detail "vmail home directory created (/var/vmail)"

create_user postfix postfix

# Postfix mail queue group
if ! getent group postdrop >/dev/null 2>&1; then
    groupadd -r postdrop
    log_detail "Created group postdrop"
fi

create_user dovecot dovecot
create_user dovenull dovecot
create_user proftpd proftpd
create_user clamav clamav

# www-data (usually already exists)
if ! getent group www-data >/dev/null 2>&1; then
    groupadd -r www-data
fi
if ! id www-data >/dev/null 2>&1; then
    useradd -r -g www-data -s /sbin/nologin www-data
fi

# panelica group (for ClamAV etc.)
if ! getent group panelica >/dev/null 2>&1; then
    groupadd -r panelica
    log_detail "Created group panelica"
fi

# panelica-web group (shared group for PHP-FPM users to access Redis socket etc.)
if ! getent group panelica-web >/dev/null 2>&1; then
    groupadd -r panelica-web
    log_detail "Created group panelica-web"
fi

# SSH jail groups
for grp in sshjailed sshfull; do
    if ! getent group "${grp}" >/dev/null 2>&1; then
        groupadd -r "${grp}"
        log_detail "Created group ${grp}"
    fi
done

# 'nogroup' — a base group on Debian/Ubuntu, absent on RHEL family. The bundled
# nginx (Debian-built) resolves it as its worker group; without it nginx -t
# fails with getgrnam("nogroup"). Create it for RHEL parity (no-op on Debian).
if ! getent group nogroup >/dev/null 2>&1; then
    groupadd -r nogroup
    log_detail "Created group nogroup (RHEL parity for bundled nginx)"
fi

log_info "All system users and groups are ready"

# =============================================================================
# STEP 5: Extract Package
# =============================================================================
log_step 5 ${TOTAL_STEPS} "Extracting Panelica Package"

if [ -n "${LOCAL_TARBALL}" ]; then
    if [ ! -f "${LOCAL_TARBALL}" ]; then
        log_error "Tarball not found: ${LOCAL_TARBALL}"
        exit 1
    fi
    TARBALL_PATH="${LOCAL_TARBALL}"
    TARBALL_SIZE=$(du -sh "${TARBALL_PATH}" | cut -f1)
    log_detail "Using local tarball: ${TARBALL_PATH} (${TARBALL_SIZE})"
else
    TARBALL_PATH="/tmp/panelica-latest.tar.gz"
    log_detail "Downloading from: ${DOWNLOAD_URL}"
    if command -v wget >/dev/null 2>&1; then
        wget -q --show-progress -O "${TARBALL_PATH}" "${DOWNLOAD_URL}"
    else
        curl -fSL -o "${TARBALL_PATH}" "${DOWNLOAD_URL}"
    fi
    if [ $? -ne 0 ]; then
        log_error "Failed to download tarball!"
        exit 1
    fi
    # SHA256 checksum verification
    log_detail "Verifying checksum..."
    EXPECTED_SHA=$(curl -fsSL "${CHECKSUM_URL}" | awk '{print $1}')
    ACTUAL_SHA=$(sha256sum "${TARBALL_PATH}" | awk '{print $1}')
    if [ -n "${EXPECTED_SHA}" ] && [ "${EXPECTED_SHA}" != "${ACTUAL_SHA}" ]; then
        log_error "Checksum mismatch! Expected: ${EXPECTED_SHA:0:16}... Got: ${ACTUAL_SHA:0:16}..."
        log_error "The download may be corrupted. Please try again."
        rm -f "${TARBALL_PATH}"
        exit 1
    fi
    if [ -n "${EXPECTED_SHA}" ]; then
        log_detail "Checksum verified: ${ACTUAL_SHA:0:16}..."
    else
        log_warn "Could not fetch checksum file, skipping verification"
    fi
fi

run_with_spinner "Extracting files to ${INSTALL_DIR}..." tar -xzf "${TARBALL_PATH}" -C /opt/
if [ $? -ne 0 ]; then
    log_error "Failed to extract tarball!"
    exit 1
fi

FILE_COUNT=$(find "${INSTALL_DIR}" -type f 2>/dev/null | wc -l)
DIR_SIZE=$(du -sh "${INSTALL_DIR}" 2>/dev/null | cut -f1)
log_info "Package extracted: ${FILE_COUNT} files, ${DIR_SIZE} total"

# Auto-detect version from release-info.json (jq installed in Step 2)
RELEASE_INFO="${INSTALL_DIR}/var/release-info.json"
if [ -f "${RELEASE_INFO}" ]; then
    EXTRACTED_VERSION=$(jq -r '.version // empty' "${RELEASE_INFO}" 2>/dev/null)
    if [ -n "${EXTRACTED_VERSION}" ] && [ -z "${PANELICA_VERSION}" ]; then
        PANELICA_VERSION="${EXTRACTED_VERSION}"
        log_detail "Version detected from release-info.json: ${PANELICA_VERSION}"
    fi
fi
[ -z "${PANELICA_VERSION}" ] && PANELICA_VERSION="unknown"

# --- Critical file integrity check ---
_verify_critical_files() {
    local missing=0
    local checked=0
    local critical_files=(
        # Backend binary
        "bin/panelica-server"
        "bin/pn-service"
        # Frontend
        "var/www/panel/index.html"
        # Services - core binaries
        "services/nginx-panel/sbin/nginx"
        "services/nginx-customer/sbin/nginx"
        "services/apache/bin/httpd"
        "services/postgresql/bin/postgres"
        "services/mysql/bin/mysqld"
        "services/redis/bin/redis-server"
        "services/postfix/sbin/postfix"
        "services/dovecot/sbin/dovecot"
        "services/proftpd/sbin/proftpd"
        # PHP (at least one version)
        "services/php/8.3/bin/php"
        "services/php/8.3/sbin/php-fpm"
        # phpMyAdmin critical vendor packages
        "services/phpmyadmin/index.php"
        "services/phpmyadmin/vendor/autoload.php"
        "services/phpmyadmin/vendor/symfony/cache/Adapter/ArrayAdapter.php"
        "services/phpmyadmin/vendor/psr/log/Psr/Log/LoggerInterface.php"
        # Config templates
        "etc/templates/nginx/vhost.tpl"
        # Systemd
        "etc/systemd/panelica-backend.service"
    )

    for f in "${critical_files[@]}"; do
        if [ ! -f "${INSTALL_DIR}/${f}" ]; then
            log_warn "MISSING: ${f}"
            missing=$((missing + 1))
        fi
        checked=$((checked + 1))
    done

    if [ ${missing} -gt 0 ]; then
        log_error "${missing}/${checked} critical files missing from tarball! Release may be corrupted."
        log_error "Re-run build-release.sh and try again."
        exit 1
    fi
    log_detail "Verified ${checked} critical files — all present"
}
run_with_spinner "Verifying critical files..." _verify_critical_files

# Clean up tarball after successful extraction + verification
if [ -f "${TARBALL_PATH}" ]; then
    rm -f "${TARBALL_PATH}"
    log_detail "Tarball removed: ${TARBALL_PATH}"
fi

# =============================================================================
# STEP 6: Create Directory Structure
# =============================================================================
log_step 6 ${TOTAL_STEPS} "Creating Directory Structure"

# Databases
mkdir -p ${INSTALL_DIR}/var/db/{postgresql,mysql,redis}
mkdir -p ${INSTALL_DIR}/var/run/{postgresql,mysqld,clamav,apache,dovecot}

# Logs
mkdir -p ${INSTALL_DIR}/var/logs/{backend,nginx-panel,nginx-customer,nginx,apache,php-fpm,mysql,postgresql,redis,dns,mail,proftpd,clamav,dovecot,bandwidth,cgroup,external-api,frontend,resource-alerts,roundcube,builder,cron-scheduler}

# PHP-FPM log directories (each version)
for phpver in 8.1 8.2 8.3 8.4 8.5; do
    mkdir -p ${INSTALL_DIR}/services/php/${phpver}/var/log
done

# Nginx customer logs
mkdir -p ${INSTALL_DIR}/services/nginx-customer/logs
touch ${INSTALL_DIR}/services/nginx-customer/logs/error.log
touch ${INSTALL_DIR}/services/nginx-customer/logs/access.log

# Cache and misc
mkdir -p ${INSTALL_DIR}/var/{ssl,tmp,cache/bind,dns/zones,namespaces,chroot,license,spool/postfix/private,run/bind}
mkdir -p ${INSTALL_DIR}/etc/ssl/panel
mkdir -p ${INSTALL_DIR}/services/apache/logs
mkdir -p ${INSTALL_DIR}/tmp/phpmyadmin/{temp,save,upload}
mkdir -p ${INSTALL_DIR}/services/roundcube/{temp,logs}
mkdir -p ${INSTALL_DIR}/var/{mail,ftp,quarantine,trash,uploads,exports,avatars,proftpd,clamav,pgadmin,backups/database,snapshots,backup,wal}
mkdir -p ${INSTALL_DIR}/services/pgadmin4/{logs,sessions,data,storage}
mkdir -p ${INSTALL_DIR}/lib/bpf
mkdir -p ${INSTALL_DIR}/var/lib/php/sessions
chmod 1733 ${INSTALL_DIR}/var/lib/php/sessions
mkdir -p ${INSTALL_DIR}/var/lib/allowed-cmds

# Customer nginx
mkdir -p ${INSTALL_DIR}/etc/customer-nginx/vhosts.d
mkdir -p ${INSTALL_DIR}/etc/panel-nginx/conf.d
mkdir -p ${INSTALL_DIR}/services/nginx-customer/html

# Nginx temp dirs (must be owned by www-data - nginx worker user)
mkdir -p ${INSTALL_DIR}/services/nginx-customer/{client_body_temp,proxy_temp,fastcgi_temp,scgi_temp,uwsgi_temp}
chown -R www-data:root ${INSTALL_DIR}/services/nginx-customer/{client_body_temp,proxy_temp,fastcgi_temp,scgi_temp,uwsgi_temp}

# Apache vhosts
mkdir -p ${INSTALL_DIR}/etc/apache/vhosts.d
mkdir -p ${INSTALL_DIR}/etc/apache/subdomains

# PHP-FPM users directory
mkdir -p ${INSTALL_DIR}/etc/php-fpm-users

log_info "Directory structure created (23 log dirs, 8 runtime dirs)"

# =============================================================================
# STEP 7: Set File Permissions
# =============================================================================
log_step 7 ${TOTAL_STEPS} "Setting File Permissions"

# Special permissions
chmod 1777 ${INSTALL_DIR}/tmp
chmod 700 ${INSTALL_DIR}/var/license
chmod 775 ${INSTALL_DIR}/var/run

# Service ownership
chown -R postgres:postgres ${INSTALL_DIR}/var/{db/postgresql,run/postgresql,logs/postgresql}
chown -R mysql:mysql ${INSTALL_DIR}/var/{db/mysql,run/mysqld,logs/mysql}
chown -R redis:redis ${INSTALL_DIR}/var/{db/redis,logs/redis}
chown -R bind:bind ${INSTALL_DIR}/var/{cache/bind,dns,run/bind}
chown -R www-data:www-data ${INSTALL_DIR}/var/logs/php-fpm ${INSTALL_DIR}/tmp/phpmyadmin ${INSTALL_DIR}/services/roundcube/{temp,logs}
chown -R clamav:clamav ${INSTALL_DIR}/var/{run/clamav,logs/clamav,quarantine,clamav}
chown -R clamav:clamav ${INSTALL_DIR}/services/clamav 2>/dev/null || true
chown postfix:postfix ${INSTALL_DIR}/var/spool/postfix/private
chown -R dovecot:dovecot ${INSTALL_DIR}/var/run/dovecot 2>/dev/null || true

# Bind config directory
if [ -d "${INSTALL_DIR}/etc/bind" ]; then
    chown -R bind:bind ${INSTALL_DIR}/etc/bind
fi

# ClamAV panelica group membership
usermod -aG panelica clamav 2>/dev/null || true

log_info "File permissions and ownership configured"

# =============================================================================
# STEP 8: Generate Configuration
# =============================================================================
log_step 8 ${TOTAL_STEPS} "Generating Configuration (panelica.conf)"

# Generate random passwords (no newlines!)
PG_PASS=""  # PostgreSQL uses trust auth on local socket
MYSQL_PASS=$(openssl rand -base64 32 | tr -d '/+=\n' | head -c 32)
REDIS_PASS=$(openssl rand -base64 32 | tr -d '/+=\n' | head -c 32)
JWT_SECRET=$(openssl rand -base64 64 | tr -d '/+=\n' | head -c 64)
ENCRYPTION_KEY=$(openssl rand -hex 32 | head -c 64)
PANEL_PORT=8443

log_detail "Generated secure random passwords for MySQL, Redis, JWT"

cat > ${INSTALL_DIR}/panelica.conf << CONFEOF
# =============================================================================
# Panelica Server Panel Configuration
# Generated: $(date -u '+%Y-%m-%d %H:%M:%S UTC')
# Version: ${PANELICA_VERSION}
# =============================================================================

[database.postgresql]
host = ${INSTALL_DIR}/var/run/postgresql
port = 5433
user = postgres
password =
database = panelica_dev
sslmode = disable
max_open_conns = 50
max_idle_conns = 10
conn_max_lifetime = 3600

[database.mysql]
socket = ${INSTALL_DIR}/var/run/mysqld/mysqld.sock
user = root
password = ${MYSQL_PASS}

[cache.redis]
host = 127.0.0.1
port = 6479
password = ${REDIS_PASS}
socket = ${INSTALL_DIR}/var/run/redis.sock
database = 0

[security]
encryption_key = ${ENCRYPTION_KEY}
jwt_secret = ${JWT_SECRET}
session_timeout = 900
refresh_token_lifetime = 604800

[panel]
root_username = root
root_password = SETUP_WIZARD_WILL_SET
root_email = root@${SERVER_HOSTNAME}
panel_name = Panelica Server Panel
company_name = Panelica
support_email = support@panelica.com
support_url = https://panelica.com/support

[server]
primary_ip = ${SERVER_IP}
hostname = ${SERVER_HOSTNAME}
nameserver1 = ns1.panelica.com
nameserver2 = ns2.panelica.com

[backup]
enabled = true
schedule = 0 3 * * *
retention_days = 10
backup_directory = ${INSTALL_DIR}/var/backups/database

[email]
smtp_host = localhost
smtp_port = 25
smtp_from = noreply@${SERVER_HOSTNAME}
smtp_from_name = Panelica Panel

[logging]
level = info
directory = ${INSTALL_DIR}/var/logs
max_size_mb = 100
max_backups = 10
max_age_days = 30
compress = true

[api]
listen_address = 0.0.0.0
listen_port = 3001
enable_https = true
ssl_cert = ${INSTALL_DIR}/var/ssl/backend.crt
ssl_key = ${INSTALL_DIR}/var/ssl/backend.key
rate_limit_enabled = true
rate_limit_requests = 100
rate_limit_window = 60
cors_enabled = true
cors_allowed_origins = https://${SERVER_IP}:${PANEL_PORT}, https://${SERVER_HOSTNAME}:${PANEL_PORT}, https://localhost:${PANEL_PORT}

[frontend]
listen_port = ${PANEL_PORT}
enable_https = true

[paths]
services_root = ${INSTALL_DIR}/services
apache_path = ${INSTALL_DIR}/services/apache
nginx_customer_path = ${INSTALL_DIR}/services/nginx-customer
nginx_panel_path = ${INSTALL_DIR}/services/nginx-panel
php_path = ${INSTALL_DIR}/services/php
postgresql_path = ${INSTALL_DIR}/services/postgresql
mysql_path = ${INSTALL_DIR}/services/mysql
redis_path = ${INSTALL_DIR}/services/redis
postfix_path = ${INSTALL_DIR}/services/postfix
dovecot_path = ${INSTALL_DIR}/services/dovecot
var_root = ${INSTALL_DIR}/var
etc_root = ${INSTALL_DIR}/etc
tmp_root = ${INSTALL_DIR}/tmp

[phpmyadmin]
apache_enabled = true
apache_port = 7081
apache_listen_address = 127.0.0.1
php_fpm_pool = phpmyadmin
php_version = 8.3
php_fpm_socket = ${INSTALL_DIR}/var/run/php-fpm-phpmyadmin.sock
upload_max_filesize = 512M
post_max_size = 512M
max_execution_time = 600
memory_limit = 512M

[pgadmin]
enabled = true
admin_email = admin@local.dev
admin_password = admin123
listen_address = 127.0.0.1
listen_port = 5050

[license]
central_url = ${CENTRAL_URL}
license_key =
secret_key =
heartbeat_interval = 21600
grace_period = 172800
public_key = xLOBSX99A5Pgc/Dz73iyCaSftoSvAXfoONFiUin4QsU=
key_id = key_20260123_233430_8e621c38
auto_update_check = true
CONFEOF

chmod 600 ${INSTALL_DIR}/panelica.conf
log_info "Configuration generated with secure random credentials"

# =============================================================================
# STEP 9: Configure Library Paths
# =============================================================================
log_step 9 ${TOTAL_STEPS} "Configuring Library Paths"

cat > /etc/ld.so.conf.d/panelica.conf << 'EOF'
/opt/panelica/lib
/opt/panelica/lib/compat
/opt/panelica/services/postgresql/lib
/opt/panelica/services/mysql/lib
EOF

# Library compatibility symlinks (Ubuntu 24.04+, Debian 12+ — time64 ABI transition)
# Multiarch lib dir differs: Debian /usr/lib/x86_64-linux-gnu, RHEL /usr/lib64.
# The t64/MagickWand symlink logic below is Debian-specific and no-ops on RHEL
# (the t64 source files do not exist there).
if [ "${OS_FAMILY}" = "rhel" ]; then
    LIB_DIR="/usr/lib64"
else
    LIB_DIR="/usr/lib/x86_64-linux-gnu"
fi

# libaio.so.1 was renamed to libaio.so.1t64 (ABI compatible)
if [ ! -f "${LIB_DIR}/libaio.so.1" ] && [ -f "${LIB_DIR}/libaio.so.1t64" ]; then
    ln -sf "${LIB_DIR}/libaio.so.1t64" "${LIB_DIR}/libaio.so.1"
    log_detail "Created libaio.so.1 symlink"
fi

# NOTE: PHP imagick.so is self-contained — it links libMagick{Wand,Core}-6.Q16.so.7
# bundled in services/php/<ver>/runtime/ and resolves them via relative RPATH.
# No system ImageMagick library or .so.6 symlink is needed; the old .so.6 shim
# that lived here was obsolete and produced a misleading "not found" warning.

# Debian compat: libraries compiled on Ubuntu that have different names on Debian
# lib/compat/ ships these pre-built — verify they resolve via ldconfig
COMPAT_DIR="${INSTALL_DIR}/lib/compat"
_compat_check() {
    local soname="$1"
    if ! ldconfig -p 2>/dev/null | grep -q "${soname}"; then
        if [ -f "${COMPAT_DIR}/${soname}" ]; then
            log_detail "Compat library active: ${soname}"
        else
            log_detail "Warning: ${soname} not found (some features may be unavailable)"
        fi
    fi
}
_compat_check "libpcre.so.3"
_compat_check "libtidy.so.5deb1"
_compat_check "libjpeg.so.8"
_compat_check "libc-client.so.2007e"
_compat_check "libodbc.so.2"

# Disable PHP extensions that require libraries not available on Debian
# (imap needs libc-client2007e, odbc/pdo_odbc need libodbc2 — not packaged on Debian 13)
if [ "${OS_ID}" = "debian" ]; then
    log_detail "Checking PHP extensions for Debian compatibility..."
    _disable_ext_if_broken() {
        local ver="$1" ext="$2"
        local extdir="${INSTALL_DIR}/services/php/${ver}/lib/php/extensions"
        local confdir="${INSTALL_DIR}/services/php/${ver}/etc/conf.d"
        local ini="${confdir}/${ext}.ini"
        [ -f "$ini" ] || return 0
        # Find the .so file
        local sofile
        sofile=$(find "$extdir" -name "${ext}.so" 2>/dev/null | head -1)
        [ -f "$sofile" ] || return 0
        # Check if all shared libs resolve
        if ldd "$sofile" 2>&1 | grep -q "not found"; then
            mv "$ini" "${ini}.disabled"
            log_detail "Disabled PHP ${ver} extension ${ext} (missing shared library)"
        fi
    }
    for phpver in 8.1 8.2 8.3 8.4; do
        [ -d "${INSTALL_DIR}/services/php/${phpver}" ] || continue
        for ext in imap odbc pdo_odbc; do
            _disable_ext_if_broken "$phpver" "$ext"
        done
    done
fi

ldconfig 2>> "${LOG_FILE}"

log_info "Shared library paths registered (ldconfig updated)"

# =============================================================================
# STEP 10: Create System Symlinks
# =============================================================================
log_step 10 ${TOTAL_STEPS} "Creating System Symlinks"

# PostgreSQL share (required for initdb)
mkdir -p /usr/share/postgresql
ln -sf ${INSTALL_DIR}/services/postgresql/share /usr/share/postgresql/17
ln -sf ${INSTALL_DIR}/services/postgresql/share /usr/share/postgresql/18
log_detail "PostgreSQL share directory linked"

# MySQL socket compatibility
ln -sf ${INSTALL_DIR}/var/run/mysqld/mysqld.sock ${INSTALL_DIR}/var/run/mysqld.sock 2>/dev/null || true
mkdir -p /tmp
ln -sf ${INSTALL_DIR}/var/run/mysqld/mysqld.sock /tmp/mysql.sock 2>/dev/null || true
log_detail "MySQL socket symlinks created"

# Postfix config
ln -sf ${INSTALL_DIR}/etc/postfix /etc/postfix 2>/dev/null || true
log_detail "Postfix configuration linked"

# Builder symlink
if [ -f "${INSTALL_DIR}/builder/pn-builder" ]; then
    ln -sf ${INSTALL_DIR}/builder/pn-builder ${INSTALL_DIR}/bin/pn-builder 2>/dev/null || true
    log_detail "Builder binary linked to bin/"
fi

log_info "All symlinks created"

# =============================================================================
# STEP 11: Generate SSL Certificates
# =============================================================================
log_step 11 ${TOTAL_STEPS} "Generating SSL Certificates"

# Backend API SSL (self-signed, 10 years)
log_detail "Generating backend API certificate..."
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
    -keyout ${INSTALL_DIR}/var/ssl/backend.key \
    -out ${INSTALL_DIR}/var/ssl/backend.crt \
    -subj "/C=TR/ST=Istanbul/L=Istanbul/O=Panelica/CN=localhost" \
    2>/dev/null
chmod 600 ${INSTALL_DIR}/var/ssl/backend.key
log_detail "Backend API SSL certificate ready"

# Panel HTTPS SSL (self-signed, 10 years)
log_detail "Generating panel HTTPS certificate..."
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
    -keyout ${INSTALL_DIR}/etc/ssl/panel/panel.key \
    -out ${INSTALL_DIR}/etc/ssl/panel/panel.crt \
    -subj "/C=TR/ST=Istanbul/L=Istanbul/O=Panelica/CN=${SERVER_HOSTNAME}" \
    2>/dev/null
chmod 600 ${INSTALL_DIR}/etc/ssl/panel/panel.key
log_detail "Panel HTTPS certificate ready"

# General Panelica SSL (for Dovecot, Postfix, etc.)
log_detail "Generating service SSL certificate..."
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
    -keyout ${INSTALL_DIR}/etc/ssl/panelica.key \
    -out ${INSTALL_DIR}/etc/ssl/panelica.crt \
    -subj "/C=TR/ST=Istanbul/L=Istanbul/O=Panelica/CN=${SERVER_HOSTNAME}" \
    2>/dev/null
chmod 600 ${INSTALL_DIR}/etc/ssl/panelica.key
log_detail "Service SSL certificate ready"

# Mail SSL cert (Dovecot/Postfix use mail.crt/mail.key)
log_detail "Generating mail SSL certificate..."
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
    -keyout ${INSTALL_DIR}/etc/ssl/mail.key \
    -out ${INSTALL_DIR}/etc/ssl/mail.crt \
    -subj "/C=TR/ST=Istanbul/L=Istanbul/O=Panelica/CN=mail.${SERVER_HOSTNAME}" \
    2>/dev/null
chmod 600 ${INSTALL_DIR}/etc/ssl/mail.key
log_detail "Mail SSL certificate ready"

# Default SSL cert for nginx-customer (catch-all server block)
log_detail "Generating default SSL certificate..."
mkdir -p ${INSTALL_DIR}/etc/ssl/default
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
    -keyout ${INSTALL_DIR}/etc/ssl/default/default.key \
    -out ${INSTALL_DIR}/etc/ssl/default/default.crt \
    -subj "/C=TR/ST=Istanbul/L=Istanbul/O=Panelica/CN=default" \
    2>/dev/null
chmod 600 ${INSTALL_DIR}/etc/ssl/default/default.key
log_detail "Default SSL certificate ready"

# DH parameters (required by Dovecot)
run_with_spinner "Generating Diffie-Hellman parameters (may take 1-2 min)..." openssl dhparam -out ${INSTALL_DIR}/etc/ssl/dh.pem 2048
log_info "All SSL certificates generated (4 certs + DH params)"

# =============================================================================
# STEP 12: Initialize PostgreSQL
# =============================================================================
log_step 12 ${TOTAL_STEPS} "Initializing PostgreSQL Database"

PG_BIN="${INSTALL_DIR}/services/postgresql/bin"
PG_DATA="${INSTALL_DIR}/var/db/postgresql"

# initdb
if [ ! -f "${PG_DATA}/PG_VERSION" ]; then
    # Use -L to explicitly set share directory (don't rely on compiled-in path)
    PG_SHARE="${INSTALL_DIR}/services/postgresql/share"
    if [ ! -f "${PG_SHARE}/pg_hba.conf.sample" ]; then
        log_error "CRITICAL: PostgreSQL share files missing (pg_hba.conf.sample not found)"
        log_error "The tarball may have been built with an overly aggressive exclude pattern"
        log_error "Cannot continue without PostgreSQL initialization files"
        exit 1
    fi
    run_with_spinner "Initializing PostgreSQL data directory..." runuser -u postgres -- ${PG_BIN}/initdb -D ${PG_DATA} -E UTF8 --locale=C -L "${PG_SHARE}"
    if [ $? -ne 0 ] || [ ! -f "${PG_DATA}/PG_VERSION" ]; then
        log_error "CRITICAL: PostgreSQL initdb failed — cannot continue"
        log_error "Check ${LOG_FILE} for details"
        exit 1
    fi

    # Custom postgresql.conf settings
    cat >> ${PG_DATA}/postgresql.conf << PGEOF

# Panelica custom settings
port = 5433
unix_socket_directories = '${INSTALL_DIR}/var/run/postgresql'
listen_addresses = '127.0.0.1'
log_directory = '${INSTALL_DIR}/var/logs/postgresql'
logging_collector = on
log_filename = 'postgresql-%Y-%m-%d.log'
PGEOF

    # pg_hba.conf - trust auth for local connections
    cat > ${PG_DATA}/pg_hba.conf << HBAEOF
# TYPE  DATABASE    USER    ADDRESS     METHOD
local   all         all                 trust
host    all         all     127.0.0.1/32 trust
host    all         all     ::1/128     trust
HBAEOF

    chown -R postgres:postgres ${PG_DATA}
    log_detail "PostgreSQL data directory initialized"
else
    log_warn "PostgreSQL data directory already exists, skipping initdb"
fi

# CRITICAL: Fix PostgreSQL data directory permissions (must be 700, not 755)
# rsync from tarball may set wrong permissions depending on source OS
chmod 700 ${PG_DATA}
chown -R postgres:postgres ${PG_DATA}

# Start PostgreSQL temporarily
log_detail "Starting PostgreSQL..."
runuser -u postgres -- ${PG_BIN}/pg_ctl -D ${PG_DATA} -l ${INSTALL_DIR}/var/logs/postgresql/startup.log start >> "${LOG_FILE}" 2>&1
sleep 3

# Verify PostgreSQL is actually running
if ! runuser -u postgres -- ${PG_BIN}/pg_isready -h ${INSTALL_DIR}/var/run/postgresql -p 5433 >> "${LOG_FILE}" 2>&1; then
    log_error "CRITICAL: PostgreSQL failed to start — cannot continue"
    log_error "Check: ${INSTALL_DIR}/var/logs/postgresql/startup.log"
    exit 1
fi

# Create databases
log_detail "Creating databases and users..."
runuser -u postgres -- ${PG_BIN}/psql -h ${INSTALL_DIR}/var/run/postgresql -p 5433 >> "${LOG_FILE}" 2>&1 << 'SQLEOF'
CREATE DATABASE panelica_dev;
CREATE DATABASE roundcube;
CREATE USER roundcube_user WITH PASSWORD 'roundcube_pass_2025';
ALTER DATABASE roundcube OWNER TO roundcube_user;
SQLEOF

# pgcrypto extension
runuser -u postgres -- ${PG_BIN}/psql -h ${INSTALL_DIR}/var/run/postgresql -p 5433 -d panelica_dev -c "CREATE EXTENSION IF NOT EXISTS pgcrypto;" >> "${LOG_FILE}" 2>&1

# Roundcube tables
if [ -f "${INSTALL_DIR}/services/roundcube/SQL/postgres.initial.sql" ]; then
    if runuser -u postgres -- ${PG_BIN}/psql -h ${INSTALL_DIR}/var/run/postgresql -p 5433 -d roundcube \
        -f ${INSTALL_DIR}/services/roundcube/SQL/postgres.initial.sql >> "${LOG_FILE}" 2>&1; then
        log_detail "Roundcube database tables created"
    else
        log_warn "Roundcube table creation had errors (may already exist) - check ${LOG_FILE}"
    fi
    if runuser -u postgres -- ${PG_BIN}/psql -h ${INSTALL_DIR}/var/run/postgresql -p 5433 -d roundcube >> "${LOG_FILE}" 2>&1 << 'RCEOF'
GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO roundcube_user;
GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO roundcube_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO roundcube_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO roundcube_user;
RCEOF
    then
        log_detail "Roundcube database permissions applied"
    else
        log_warn "Roundcube permission grants failed - check ${LOG_FILE}"
    fi
fi

# Stop PostgreSQL (will be restarted by systemd)
runuser -u postgres -- ${PG_BIN}/pg_ctl -D ${PG_DATA} stop >> "${LOG_FILE}" 2>&1
sleep 2

log_info "PostgreSQL initialized (3 databases, 2 roles, pgcrypto extension)"

# =============================================================================
# STEP 13: Configure Redis
# =============================================================================
log_step 13 ${TOTAL_STEPS} "Configuring Redis"

REDIS_CONF="${INSTALL_DIR}/etc/redis/redis.conf"
if [ -f "${REDIS_CONF}" ]; then
    # Sync generated password into Redis config
    if grep -q "^requirepass" "${REDIS_CONF}"; then
        sed -i "s|^requirepass.*|requirepass ${REDIS_PASS}|" "${REDIS_CONF}"
    else
        echo "requirepass ${REDIS_PASS}" >> "${REDIS_CONF}"
    fi

    # Clean any stale persistent data from packaging
    rm -f ${INSTALL_DIR}/var/db/redis/dump.rdb 2>/dev/null || true
    rm -f ${INSTALL_DIR}/var/db/redis/appendonly.aof 2>/dev/null || true
    rm -f ${INSTALL_DIR}/var/db/redis/appendonlydir/* 2>/dev/null || true

    log_info "Redis configured with generated password"
else
    log_warn "Redis config not found: ${REDIS_CONF}"
fi

# Ensure system Redis (port 6379) is running for customer applications
# This is separate from Panelica Redis (port 6479) which is isolated
# Debian package ships service 'redis-server'; RHEL package ships 'redis'
if [ "${OS_FAMILY}" = "rhel" ]; then
    SYS_REDIS_SVC="redis"
else
    SYS_REDIS_SVC="redis-server"
fi
if systemctl is-enabled "${SYS_REDIS_SVC}" >/dev/null 2>&1; then
    systemctl restart "${SYS_REDIS_SVC}" 2>/dev/null || true
    log_info "System Redis (port 6379) enabled for customer applications"
else
    systemctl enable "${SYS_REDIS_SVC}" 2>/dev/null || true
    systemctl start "${SYS_REDIS_SVC}" 2>/dev/null || true
    log_info "System Redis (port 6379) enabled and started for customer applications"
fi

# =============================================================================
# STEP 14: Initialize MySQL
# =============================================================================
log_step 14 ${TOTAL_STEPS} "Initializing MySQL Database"

MYSQL_BIN="${INSTALL_DIR}/services/mysql/bin"
MYSQL_DATA="${INSTALL_DIR}/var/db/mysql"

if [ ! -d "${MYSQL_DATA}/mysql" ]; then
    run_with_spinner "Initializing MySQL data directory..." ${MYSQL_BIN}/mysqld --initialize-insecure --user=mysql --basedir=${INSTALL_DIR}/services/mysql --datadir=${MYSQL_DATA}
    log_detail "MySQL data directory initialized"
fi

# Start MySQL temporarily
log_detail "Starting MySQL..."
${MYSQL_BIN}/mysqld \
    --user=mysql \
    --basedir=${INSTALL_DIR}/services/mysql \
    --datadir=${MYSQL_DATA} \
    --socket=${INSTALL_DIR}/var/run/mysqld/mysqld.sock \
    --pid-file=${INSTALL_DIR}/var/run/mysqld/mysqld.pid \
    --port=3306 \
    --log-error=${INSTALL_DIR}/var/logs/mysql/error.log \
    --daemonize >> "${LOG_FILE}" 2>&1

sleep 3

# Set root password and create phpMyAdmin user
log_detail "Configuring MySQL users..."
${MYSQL_BIN}/mysql -u root -S ${INSTALL_DIR}/var/run/mysqld/mysqld.sock >> "${LOG_FILE}" 2>&1 << MYSQLEOF
ALTER USER 'root'@'localhost' IDENTIFIED BY '${MYSQL_PASS}';
CREATE DATABASE IF NOT EXISTS phpmyadmin;
CREATE USER IF NOT EXISTS 'pma'@'localhost' IDENTIFIED BY 'pmacontrol2025';
GRANT SELECT, INSERT, UPDATE, DELETE ON phpmyadmin.* TO 'pma'@'localhost';
FLUSH PRIVILEGES;
MYSQLEOF

# phpMyAdmin control tables
if [ -f "${INSTALL_DIR}/services/phpmyadmin/sql/create_tables.sql" ]; then
    ${MYSQL_BIN}/mysql -u root -p"${MYSQL_PASS}" \
        -S ${INSTALL_DIR}/var/run/mysqld/mysqld.sock \
        phpmyadmin < ${INSTALL_DIR}/services/phpmyadmin/sql/create_tables.sql 2>/dev/null || true
    log_detail "phpMyAdmin control tables created"
fi

# Shutdown MySQL (will be restarted by systemd)
${MYSQL_BIN}/mysqladmin -u root -p"${MYSQL_PASS}" \
    -S ${INSTALL_DIR}/var/run/mysqld/mysqld.sock shutdown 2>/dev/null
sleep 2

log_info "MySQL initialized (root password set, phpMyAdmin configured)"

# RAM-based MySQL tuning (override tarball defaults based on server capacity)
MYCNF="${INSTALL_DIR}/services/mysql/my.cnf"
if [ -f "${MYCNF}" ]; then
    if [ "${TOTAL_RAM_MB}" -le 2048 ]; then
        POOL_SIZE="256M"; MAX_CONN=50
    elif [ "${TOTAL_RAM_MB}" -le 4096 ]; then
        POOL_SIZE="512M"; MAX_CONN=100
    elif [ "${TOTAL_RAM_MB}" -le 8192 ]; then
        POOL_SIZE="1G"; MAX_CONN=150
    elif [ "${TOTAL_RAM_MB}" -le 16384 ]; then
        POOL_SIZE="2G"; MAX_CONN=200
    elif [ "${TOTAL_RAM_MB}" -le 32768 ]; then
        POOL_SIZE="4G"; MAX_CONN=300
    else
        POOL_SIZE="8G"; MAX_CONN=500
    fi
    sed -i "s/^innodb_buffer_pool_size.*/innodb_buffer_pool_size = ${POOL_SIZE}/" "${MYCNF}"
    sed -i "s/^max_connections.*/max_connections = ${MAX_CONN}/" "${MYCNF}"
    log_detail "MySQL tuned for ${TOTAL_RAM_MB}MB RAM (buffer_pool=${POOL_SIZE}, max_conn=${MAX_CONN})"
fi

# =============================================================================
# STEP 15: Install Systemd Services
# =============================================================================
log_step 15 ${TOTAL_STEPS} "Installing Systemd Services"

# Copy service files
cp ${INSTALL_DIR}/etc/systemd/*.service /etc/systemd/system/ 2>/dev/null || true

# Copy slice files (cgroups v2 resource isolation)
cp ${INSTALL_DIR}/etc/systemd/*.slice /etc/systemd/system/ 2>/dev/null || true

systemctl daemon-reload

SVC_COUNT=$(ls ${INSTALL_DIR}/etc/systemd/*.service 2>/dev/null | wc -l)
log_detail "Installed ${SVC_COUNT} service files"

# Enable services
SERVICES=(
    redis postgresql mysql
    nginx-panel nginx-customer apache
    php-fpm-8.3 php-fpm-8.4
    # php-fpm-8.1 php-fpm-8.2 php-fpm-8.5 — disabled by default, enable from panel
    bind proftpd fail2ban
    # postfix dovecot — disabled by default, enable from panel when needed
    backend external-api bandwidth bandwidth-limiter ftp-quota
    cron-scheduler
)

for svc in "${SERVICES[@]}"; do
    systemctl enable "panelica-${svc}" 2>/dev/null || true
done

log_info "${#SERVICES[@]} services enabled for auto-start"

# =============================================================================
# STEP 16: Configure suEXEC
# =============================================================================
log_step 16 ${TOTAL_STEPS} "Configuring Apache suEXEC"

SUEXEC_BIN="${INSTALL_DIR}/services/apache/bin/suexec"
if [ -f "${SUEXEC_BIN}" ]; then
    chown root:www-data "${SUEXEC_BIN}"
    chmod 4750 "${SUEXEC_BIN}"
    touch ${INSTALL_DIR}/services/apache/logs/suexec_log
    chmod 644 ${INSTALL_DIR}/services/apache/logs/suexec_log
    log_info "suEXEC SUID bit set (4750 root:www-data)"
else
    log_warn "suEXEC binary not found, skipping"
fi

# =============================================================================
# STEP 17: Prepare ModSecurity
# =============================================================================
log_step 17 ${TOTAL_STEPS} "Preparing ModSecurity WAF"

if [ -d "${INSTALL_DIR}/etc/modsecurity" ]; then
    touch ${INSTALL_DIR}/etc/modsecurity/global-blacklist.txt
    touch ${INSTALL_DIR}/etc/modsecurity/bad-bots.txt
    log_info "ModSecurity blacklist files created"
fi

# =============================================================================
# STEP 18: Create PHP System Wrappers
# =============================================================================
log_step 18 ${TOTAL_STEPS} "Creating PHP System Wrappers"

PANELICA_PHP="${INSTALL_DIR}/services/php"
DEFAULT_PHP="8.4"

# Main php command (wrapper script, NOT symlink - sets PHP_INI_SCAN_DIR)
if [ -f "${PANELICA_PHP}/${DEFAULT_PHP}/bin/php" ]; then
    cat > /usr/local/bin/php << PHPEOF
#!/bin/sh
PHP_INI_SCAN_DIR=${PANELICA_PHP}/${DEFAULT_PHP}/etc/conf.d exec ${PANELICA_PHP}/${DEFAULT_PHP}/bin/php "\$@"
PHPEOF
    chmod +x /usr/local/bin/php
    log_detail "Created php wrapper (default: ${DEFAULT_PHP})"
fi

# Version-specific wrappers (php81, php82, php83, php84, php85)
for ver in 8.1 8.2 8.3 8.4 8.5; do
    short=$(echo $ver | tr -d '.')
    if [ -f "${PANELICA_PHP}/${ver}/bin/php" ]; then
        cat > /usr/local/bin/php${short} << VEOF
#!/bin/sh
PHP_INI_SCAN_DIR=${PANELICA_PHP}/${ver}/etc/conf.d exec ${PANELICA_PHP}/${ver}/bin/php "\$@"
VEOF
        chmod +x /usr/local/bin/php${short}
    fi
done
log_detail "Created version-specific wrappers (php81-php84)"

# phpize and php-config (plain symlinks are fine - they don't load extensions)
ln -sf ${PANELICA_PHP}/${DEFAULT_PHP}/bin/phpize /usr/local/bin/phpize 2>/dev/null || true
ln -sf ${PANELICA_PHP}/${DEFAULT_PHP}/bin/php-config /usr/local/bin/php-config 2>/dev/null || true
log_detail "Linked phpize and php-config"

# Composer wrapper
if [ -f "${INSTALL_DIR}/bin/composer.phar" ]; then
    cat > /usr/local/bin/composer << COMPEOF
#!/bin/sh
exec /usr/local/bin/php ${INSTALL_DIR}/bin/composer.phar "\$@"
COMPEOF
    chmod +x /usr/local/bin/composer
    log_detail "Created composer wrapper"
fi

# MySQL client wrappers + /root/.my.cnf
# Tüm mysql client binary'leri /opt/panelica/services/mysql/bin/ altında bundle'lanmış.
# Panel izole çalıştığı için /usr/local/bin'de wrapper yoksa "mysql -u root -p" gibi
# standart cPanel/Plesk komutları çalışmaz. PHP wrapper pattern'ini birebir taklit ediyoruz.
PANELICA_MYSQL="${INSTALL_DIR}/services/mysql"
PANELICA_MYSQL_SOCK="${INSTALL_DIR}/var/run/mysqld/mysqld.sock"
PANELICA_MYCNF="/root/.my.cnf"

if [ -f "${PANELICA_MYSQL}/bin/mysql" ]; then
    # 1) /root/.my.cnf — panelica.conf'tan parolayı oku, root için socket + user + password kaydet.
    #    chmod 600 ile sadece root okur. Bu sayede "mysql" tek başına direkt giriş yapar.
    _mysql_pass=$(awk '
        /^\[database.mysql\]/ {in_section=1; next}
        /^\[/ {in_section=0}
        in_section && /^[[:space:]]*password[[:space:]]*=/ {
            sub(/^[^=]*=[[:space:]]*/, "", $0)
            print $0
            exit
        }' "${INSTALL_DIR}/panelica.conf" 2>/dev/null)

    if [ -n "$_mysql_pass" ]; then
        cat > "${PANELICA_MYCNF}" << MYCNF_EOF
# Panelica-managed root client config. DO NOT EDIT BY HAND.
# Regenerated automatically by install.sh and by backend startup self-heal.
[client]
socket   = ${PANELICA_MYSQL_SOCK}
user     = root
password = ${_mysql_pass}
MYCNF_EOF
        chmod 600 "${PANELICA_MYCNF}"
        log_detail "Created /root/.my.cnf (socket + root credential)"
    else
        log_detail "WARNING: Could not read [database.mysql].password from panelica.conf — .my.cnf skipped"
    fi
    unset _mysql_pass

    # 2) /usr/local/bin wrapper'ları — 7 mysql client komutu için.
    #    PHP wrapper pattern'iyle aynı: 1 satır exec, $@ ile argümanları geçir.
    for _mybin in mysql mysqldump mysqladmin mysqlcheck mysqlimport mysqlshow mysqlbinlog; do
        if [ -f "${PANELICA_MYSQL}/bin/${_mybin}" ]; then
            cat > "/usr/local/bin/${_mybin}" << MYWRAP_EOF
#!/bin/sh
exec ${PANELICA_MYSQL}/bin/${_mybin} "\$@"
MYWRAP_EOF
            chmod +x "/usr/local/bin/${_mybin}"
        fi
    done
    log_detail "Created MySQL client wrappers (mysql, mysqldump, mysqladmin, mysqlcheck, mysqlimport, mysqlshow, mysqlbinlog)"

    # 3) mysql_config — config tool, extension build için lazım, wrapper'a gerek yok, plain symlink.
    ln -sf "${PANELICA_MYSQL}/bin/mysql_config" /usr/local/bin/mysql_config 2>/dev/null || true
fi

# Remove Xdebug from all PHP versions (causes ~164% performance degradation even in develop mode)
for _phpver_dir in "${INSTALL_DIR}"/services/php/*/; do
    [ -d "$_phpver_dir" ] || continue
    # Remove xdebug ini files (conf.d + fpm/conf.d)
    rm -f "$_phpver_dir"etc/conf.d/10-xdebug.ini "$_phpver_dir"etc/conf.d/10-xdebug.ini.disabled 2>/dev/null
    rm -f "$_phpver_dir"etc/fpm/conf.d/10-xdebug.ini "$_phpver_dir"etc/fpm/conf.d/10-xdebug.ini.disabled 2>/dev/null
    # Remove xdebug .so binary
    find "$_phpver_dir"lib/php/extensions/ -name "xdebug.so" -delete 2>/dev/null
    # Remove CLI symlinks
    _phpver=$(basename "$_phpver_dir")
    rm -f "/etc/php/${_phpver}/cli/conf.d/10-xdebug.ini" 2>/dev/null
done
log_detail "Xdebug removed from all PHP versions (production performance fix)"

# NOTE: PHP extension ini sync is handled by the backend on startup —
# SystemInitService.syncPHPExtensionConfigs() deploys the embedded
# sync_extensions.sh and runs it (hash-checked, self-healing). install.sh runs
# too early for this: the script is not on disk until the backend writes it,
# so a check here always failed and logged a misleading "not found".

log_info "PHP system wrappers created (4 versions + composer)"

# =============================================================================
# STEP 19: Prepare Customer Nginx
# =============================================================================
log_step 19 ${TOTAL_STEPS} "Preparing Customer Web Server"

# Clean any dev config remnants from all vhost directories
rm -f ${INSTALL_DIR}/etc/customer-nginx/vhosts.d/*.conf 2>/dev/null || true
rm -f ${INSTALL_DIR}/etc/nginx-customer/vhosts.d/*.conf 2>/dev/null || true
rm -f ${INSTALL_DIR}/etc/nginx-customer/subdomains/*.conf 2>/dev/null || true
rm -f ${INSTALL_DIR}/etc/apache/vhosts.d/*.conf 2>/dev/null || true
rm -f ${INSTALL_DIR}/etc/apache/subdomains/*.conf 2>/dev/null || true
mkdir -p ${INSTALL_DIR}/etc/nginx-customer/subdomains

# Clean BIND named.conf.local (dev zone entries)
if [ -f "${INSTALL_DIR}/etc/bind/named.conf.local" ]; then
    echo "// Managed by Panelica - domain zones will be added automatically" > ${INSTALL_DIR}/etc/bind/named.conf.local
    chown bind:bind ${INSTALL_DIR}/etc/bind/named.conf.local
fi

# Generate fresh rndc.key (required by named.conf controls{} for `rndc reload`).
# rndc-confgen produces matching key blocks for both rndc.key (named reads it)
# and rndc.conf (rndc client uses it). Without this, every panel-triggered
# `rndc reload <zone>` returns "connection closed" → pn-service falls back to
# full BIND restart for each new domain (DNS query downtime ~1-2s).
RNDC_KEY_FILE="${INSTALL_DIR}/etc/bind/rndc.key"
RNDC_CONF_FILE="${INSTALL_DIR}/etc/bind/rndc.conf"
if [ ! -f "${RNDC_KEY_FILE}" ] || ! grep -q '^key' "${RNDC_KEY_FILE}" 2>/dev/null; then
    log_detail "Generating BIND rndc.key (per-install unique secret)..."
    "${INSTALL_DIR}/services/bind/sbin/rndc-confgen" -a -A hmac-sha256 -k rndc-key -c "${RNDC_KEY_FILE}" >/dev/null 2>&1
    if [ -f "${RNDC_KEY_FILE}" ]; then
        chown bind:bind "${RNDC_KEY_FILE}"
        chmod 640 "${RNDC_KEY_FILE}"
        # Sync rndc.conf secret with newly generated rndc.key (must match)
        NEW_SECRET=$(grep -oP 'secret\s+"\K[^"]+' "${RNDC_KEY_FILE}" | head -1)
        if [ -n "${NEW_SECRET}" ] && [ -f "${RNDC_CONF_FILE}" ]; then
            sed -i "s|secret \"[^\"]*\"|secret \"${NEW_SECRET}\"|" "${RNDC_CONF_FILE}"
        fi
        log_detail "BIND rndc.key generated and rndc.conf secret synced"
    else
        log_warn "rndc-confgen failed; rndc reload will fall back to BIND restart"
    fi
fi

# Ensure named.conf carries an explicit rndc command channel (idempotent).
# Without this block, named falls back to its built-in search path
# (<prefix>/etc/rndc.key) which does NOT exist in our isolated layout, and
# every `rndc reload <zone>` fails with "connection to remote host closed".
# DKIM auto-publish, dynamic zone updates, and per-domain reloads all need it.
NAMED_CONF="${INSTALL_DIR}/etc/bind/named.conf"
if [ -f "${NAMED_CONF}" ] && ! grep -qE '^[[:space:]]*controls' "${NAMED_CONF}"; then
    cat >> "${NAMED_CONF}" <<NCEOF

// Panelica: explicit rndc command channel — required so rndc can reach named
include "${RNDC_KEY_FILE}";

controls {
    inet 127.0.0.1 port 953 allow { 127.0.0.1; } keys { "rndc-key"; };
};
NCEOF
    log_detail "named.conf controls block appended (rndc command channel on 127.0.0.1:953)"
fi

# Clean DNS zone files from dev
rm -f ${INSTALL_DIR}/var/dns/zones/db.* 2>/dev/null || true
log_detail "Cleaned dev configuration remnants"

# Default page
if [ ! -f "${INSTALL_DIR}/services/nginx-customer/html/index.html" ]; then
    cat > ${INSTALL_DIR}/services/nginx-customer/html/index.html << 'HTMLEOF'
<!DOCTYPE html>
<html>
<head><title>Welcome</title></head>
<body>
<h1>Domain not configured</h1>
<p>This domain has not been configured yet. Please contact the server administrator.</p>
</body>
</html>
HTMLEOF
fi

# Default error pages
mkdir -p ${INSTALL_DIR}/var/www/panel-errors
if [ ! -f "${INSTALL_DIR}/var/www/panel-errors/50x.html" ]; then
    cat > ${INSTALL_DIR}/var/www/panel-errors/50x.html << 'ERREOF'
<!DOCTYPE html>
<html><head><title>Server Error</title></head>
<body><h1>Server Error</h1><p>The server encountered an error. Please try again later.</p></body>
</html>
ERREOF
fi

log_info "Customer web server defaults configured"

# =============================================================================
# STEP 20: Configure pgAdmin4 & Misc
# =============================================================================
log_step 20 ${TOTAL_STEPS} "Configuring pgAdmin4"

# pgAdmin4: Rebuild virtualenv if Python version mismatch
# pgAdmin ships with Python 3.10 venv, but target OS may have 3.12+
PGADMIN_DIR="${INSTALL_DIR}/services/pgadmin4"
if [ -d "${PGADMIN_DIR}/lib" ]; then
    PGADMIN_PYTHON_VER=$(ls "${PGADMIN_DIR}/lib/" 2>/dev/null | head -1)  # e.g. python3.10
    SYSTEM_PYTHON_VER="python$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null || echo '3.10')"
    if [ "$PGADMIN_PYTHON_VER" != "$SYSTEM_PYTHON_VER" ] && [ -n "$PGADMIN_PYTHON_VER" ]; then
        log_detail "pgAdmin4 venv mismatch: ${PGADMIN_PYTHON_VER} vs system ${SYSTEM_PYTHON_VER}"
        # Install python venv + system gunicorn (Python 3.14 PyPI wheel not yet available on Ubuntu 26.04)
        if [ "${OS_FAMILY}" = "rhel" ]; then
            # RHEL python3 ships the venv module built-in; gunicorn via EPEL
            dnf -y install python3-pip python3-gunicorn >> "${LOG_FILE}" 2>&1 || true
        else
            apt-get install -y python3-venv python3-pip python3-gunicorn >> "${LOG_FILE}" 2>&1 || true
        fi
        # Rebuild venv with system Python, reusing existing site-packages
        OLD_SITE="${PGADMIN_DIR}/lib/${PGADMIN_PYTHON_VER}/site-packages"
        if [ -d "$OLD_SITE" ]; then
            python3 -m venv --system-site-packages "${PGADMIN_DIR}/venv_new" >> "${LOG_FILE}" 2>&1
            # Install gunicorn only — pgadmin4 is NOT a PyPI package, it is copied
            # directly from $OLD_SITE below. Listing it here just produced a noisy
            # "Could not find a version that satisfies pgadmin4" pip ERROR.
            "${PGADMIN_DIR}/venv_new/bin/pip" install --no-index --find-links="$OLD_SITE" \
                gunicorn >> "${LOG_FILE}" 2>&1 || true
            # If no-index fails (no wheels), install from PyPI using requirements
            if ! "${PGADMIN_DIR}/venv_new/bin/python3" -c "import gunicorn" 2>/dev/null; then
                log_detail "Installing pgAdmin4 dependencies from PyPI..."
                "${PGADMIN_DIR}/venv_new/bin/pip" install gunicorn >> "${LOG_FILE}" 2>&1 || true
                # Copy pgadmin4 package directly (it's not on PyPI as 'pgadmin4')
                NEW_SITE="${PGADMIN_DIR}/venv_new/lib/${SYSTEM_PYTHON_VER}/site-packages"
                cp -r "${OLD_SITE}/pgadmin4" "$NEW_SITE/" 2>/dev/null || true
                cp -r "${OLD_SITE}/pgadmin4"*.dist-info "$NEW_SITE/" 2>/dev/null || true
                # Copy all remaining packages that aren't standard lib
                for pkg_dir in "$OLD_SITE"/*/; do
                    pkg_name=$(basename "$pkg_dir")
                    [ -d "${NEW_SITE}/${pkg_name}" ] || cp -r "$pkg_dir" "$NEW_SITE/" 2>/dev/null || true
                done
                for egg in "$OLD_SITE"/*.egg-info "$OLD_SITE"/*.dist-info; do
                    [ -d "$egg" ] || continue
                    egg_name=$(basename "$egg")
                    [ -d "${NEW_SITE}/${egg_name}" ] || cp -r "$egg" "$NEW_SITE/" 2>/dev/null || true
                done
            fi
            # Replace old venv structure
            rm -rf "${PGADMIN_DIR}/lib/${PGADMIN_PYTHON_VER}"
            mkdir -p "${PGADMIN_DIR}/lib/${SYSTEM_PYTHON_VER}"
            cp -r "${PGADMIN_DIR}/venv_new/lib/${SYSTEM_PYTHON_VER}/site-packages" \
                  "${PGADMIN_DIR}/lib/${SYSTEM_PYTHON_VER}/" 2>/dev/null || true
            # Update bin symlinks
            ln -sf /usr/bin/python3 "${PGADMIN_DIR}/bin/python3"
            ln -sf python3 "${PGADMIN_DIR}/bin/python"
            rm -f "${PGADMIN_DIR}/bin/python3.10" "${PGADMIN_DIR}/bin/python3.11"
            ln -sf python3 "${PGADMIN_DIR}/bin/${SYSTEM_PYTHON_VER}"
            # Update gunicorn shebang
            sed -i "1s|.*|#!${PGADMIN_DIR}/bin/python3|" "${PGADMIN_DIR}/bin/gunicorn" 2>/dev/null || true
            # Fix start script chdir path
            sed -i "s|python3.10|${SYSTEM_PYTHON_VER}|g" "${PGADMIN_DIR}/scripts/start_pgadmin.sh" 2>/dev/null || true
            # Cleanup temp venv
            rm -rf "${PGADMIN_DIR}/venv_new"
            log_info "pgAdmin4 rebuilt for ${SYSTEM_PYTHON_VER}"
        fi
    else
        log_detail "pgAdmin4 Python version matches system (${SYSTEM_PYTHON_VER})"
    fi
fi

# =============================================================================
# STEP 21: Initialize Cgroup v2 Slices
# =============================================================================
log_step 21 ${TOTAL_STEPS} "Initializing Cgroup v2 Resource Isolation"

# Slice files already copied in Step 15
if [ -f "${INSTALL_DIR}/scripts/cgroup/init_controllers.sh" ]; then
    bash ${INSTALL_DIR}/scripts/cgroup/init_controllers.sh >> "${LOG_FILE}" 2>&1 || true
    log_info "Cgroup v2 controllers initialized"
else
    log_warn "Cgroup init script not found, skipping"
fi

# Mount BPF filesystem (required for eBPF bandwidth limiter pinned maps)
if ! mount | grep -q "bpf on /sys/fs/bpf"; then
    mkdir -p /sys/fs/bpf
    mount -t bpf bpf /sys/fs/bpf 2>/dev/null || true
    log_info "BPF filesystem mounted at /sys/fs/bpf"
fi
# Add to fstab if not present (persistent across reboots)
if ! grep -q "bpf /sys/fs/bpf" /etc/fstab 2>/dev/null; then
    echo "bpf /sys/fs/bpf bpf rw,nosuid,nodev,noexec,relatime,mode=700 0 0" >> /etc/fstab
    log_info "BPF filesystem added to /etc/fstab"
fi

# =============================================================================
# STEP 22: Verify Nginx Panel Configuration
# =============================================================================
log_step 22 ${TOTAL_STEPS} "Verifying Panel Web Server"

NGINX_PANEL_CONF="${INSTALL_DIR}/services/nginx-panel/conf/nginx.conf"
if [ -f "${NGINX_PANEL_CONF}" ]; then
    if ! grep -q "listen ${PANEL_PORT}" "${NGINX_PANEL_CONF}"; then
        log_warn "Panel port mismatch detected - Setup Wizard will correct this"
    fi

    # Ensure gzip compression is enabled (CRITICAL for panel load speed)
    if ! grep -q "gzip on" "${NGINX_PANEL_CONF}"; then
        sed -i '/sendfile on;/a\
\
    # Gzip compression (CRITICAL for performance)\
    gzip on;\
    gzip_vary on;\
    gzip_proxied any;\
    gzip_comp_level 6;\
    gzip_min_length 1024;\
    gzip_types text/plain text/css text/xml text/javascript application/javascript application/json application/xml application/rss+xml application/atom+xml image/svg+xml font/woff font/woff2;' "${NGINX_PANEL_CONF}"
        log_detail "Gzip compression enabled"
    fi

    # Ensure nginx-panel logs directory exists
    mkdir -p "${INSTALL_DIR}/services/nginx-panel/logs" 2>/dev/null || true

    log_info "Nginx panel configuration verified (port ${PANEL_PORT})"
fi

# =============================================================================
# STEP 23: Configure AppArmor Profiles
# =============================================================================
log_step 23 ${TOTAL_STEPS} "Configuring AppArmor Profiles"

# AppArmor profiles — Debian-family only. RHEL uses SELinux (install sets it to
# permissive in Step 1's RHEL path; proper SELinux policy is future work).
if [ "${OS_FAMILY}" = "debian" ]; then

# Disable ClamAV AppArmor profiles (already done in Step 3, ensure persistence)
for profile in usr.sbin.clamd usr.bin.freshclam; do
    if [ -f "/etc/apparmor.d/${profile}" ]; then
        aa-disable "${profile}" >> "${LOG_FILE}" 2>&1 || true
    fi
done

# Configure pn-ssh-shell AppArmor profile (complain mode - logs but doesn't block)
cat > /etc/apparmor.d/panelica.pn-ssh-shell << 'APPARMOR_PNS'
#include <tunables/global>

profile pn-ssh-shell /opt/panelica/bin/pn-ssh-shell flags=(attach_disconnected,complain) {
  #include <abstractions/base>
  #include <abstractions/bash>
  #include <abstractions/nameservice>
  #include <abstractions/authentication>

  capability setuid,
  capability setgid,
  capability dac_override,
  capability dac_read_search,
  capability audit_write,
  capability sys_resource,

  /opt/panelica/bin/** rix,
  /bin/bash ix,
  /usr/bin/bash ix,
  /bin/sh ix,
  /usr/bin/sh ix,
  /bin/rbash ix,
  /usr/bin/rbash ix,
  /usr/bin/** rix,
  /usr/sbin/** rix,
  /usr/libexec/** mr,
  /usr/lib/** mr,
  /lib/** mr,
  /lib64/** mr,
  /etc/** r,
  /proc/** r,
  /sys/** r,
  /dev/** rw,
  /run/** rw,
  /var/lib/sudo/** rwk,
  /tmp/** rw,
  /home/** rwlk,
  /opt/panelica/var/** rwlk,
  /opt/panelica/lib/** rix,
}
APPARMOR_PNS
apparmor_parser -r /etc/apparmor.d/panelica.pn-ssh-shell >> "${LOG_FILE}" 2>&1 || true

fi  # end AppArmor (Debian-family only)

# Register pn-ssh-shell as valid login shell
if ! grep -q "pn-ssh-shell" /etc/shells 2>/dev/null; then
    echo "/opt/panelica/bin/pn-ssh-shell" >> /etc/shells
fi
if ! grep -q "/usr/sbin/nologin" /etc/shells 2>/dev/null; then
    echo "/usr/sbin/nologin" >> /etc/shells
fi

# Configure sudoers for SSH cgroup shell
cat > /etc/sudoers.d/panelica-ssh << 'SUDOERS_SSH'
# Allow sshfull group members to run pn-cgroup-shell via sudo without password
%sshfull ALL=(root) NOPASSWD: /opt/panelica/bin/pn-cgroup-shell
SUDOERS_SSH
chmod 440 /etc/sudoers.d/panelica-ssh

# Ensure sshd privilege separation directory exists (required for sshd -t config test)
mkdir -p /run/sshd

log_info "AppArmor profiles and SSH shell configured"

# =============================================================================
# STEP 24: Set Binary Permissions
# =============================================================================
log_step 24 ${TOTAL_STEPS} "Setting Binary Permissions"

# All binaries get execute permission
chmod +x ${INSTALL_DIR}/bin/* 2>/dev/null || true

# Shell scripts
chmod +x ${INSTALL_DIR}/bin/pn-service 2>/dev/null || true
chmod +x ${INSTALL_DIR}/bin/pn-ssh-shell 2>/dev/null || true
chmod +x ${INSTALL_DIR}/bin/pn-cgroup-shell 2>/dev/null || true
chmod +x ${INSTALL_DIR}/bin/pn-cgroup-move 2>/dev/null || true
chmod +x ${INSTALL_DIR}/bin/pn-cgroup-sudo 2>/dev/null || true
chmod +x ${INSTALL_DIR}/bin/pn-namespace-init 2>/dev/null || true
chmod +x ${INSTALL_DIR}/bin/pn-namespace-enter 2>/dev/null || true
chmod +x ${INSTALL_DIR}/bin/pn-cron-exec 2>/dev/null || true
chmod +x ${INSTALL_DIR}/bin/pn-wp-cli 2>/dev/null || true

# Scripts
chmod +x ${INSTALL_DIR}/scripts/cgroup/init_controllers.sh 2>/dev/null || true
chmod +x ${INSTALL_DIR}/scripts/release/*.sh 2>/dev/null || true

BIN_COUNT=$(ls ${INSTALL_DIR}/bin/ 2>/dev/null | wc -l)
log_info "Execute permissions set on ${BIN_COUNT} binaries"

# Global CLI tool access: panelica command from anywhere
ln -sf ${INSTALL_DIR}/bin/panelica /usr/local/bin/panelica 2>/dev/null || true
ln -sf ${INSTALL_DIR}/bin/pn-service /usr/local/bin/pn-service 2>/dev/null || true
log_detail "Created CLI symlinks (/usr/local/bin/panelica, pn-service)"

# System tuning for Panelica
log_detail "Applying system kernel tuning..."

# Write all Panelica sysctl params to a dedicated drop-in file (idempotent)
# Minimal RHEL 10 images don't pre-create /etc/sysctl.d — ensure it exists.
mkdir -p /etc/sysctl.d /etc/security/limits.d
cat > /etc/sysctl.d/99-panelica.conf << 'SYSCTL'
# Panelica Server Panel - System Tuning
# Applied during install, persists across reboots

# inotify: systemd service monitoring
fs.inotify.max_user_watches = 524288
fs.inotify.max_user_instances = 512

# File descriptors: 20 services + user processes need headroom
fs.file-max = 2097152

# Network: High-traffic web server tuning
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 65535
net.core.netdev_max_backlog = 65535
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_fin_timeout = 15

# Memory: Prefer keeping data in RAM over swapping
vm.swappiness = 10
vm.dirty_ratio = 15
vm.dirty_background_ratio = 5
SYSCTL
sysctl -p /etc/sysctl.d/99-panelica.conf > /dev/null 2>&1 || true
# Clean up old sysctl.conf entries if present (from previous versions)
sed -i '/# Panelica - systemd service monitoring/d; /fs.inotify.max_user_watches=524288/d; /fs.inotify.max_user_instances=512/d' /etc/sysctl.conf 2>/dev/null || true

# File descriptor limits for Panelica services and users
cat > /etc/security/limits.d/99-panelica.conf << 'LIMITS'
# Panelica Server Panel - File Descriptor Limits
*               soft    nofile          65535
*               hard    nofile          131072
root            soft    nofile          65535
root            hard    nofile          131072
LIMITS

# Ensure PAM limits module is enabled (reads limits.d).
# Debian uses /etc/pam.d/common-session; RHEL already enables pam_limits by
# default in /etc/pam.d/system-auth — nothing to do there.
if [ "${OS_FAMILY}" = "debian" ]; then
    if ! grep -q "pam_limits.so" /etc/pam.d/common-session 2>/dev/null; then
        echo "session required pam_limits.so" >> /etc/pam.d/common-session
    fi
fi

# Detect and preserve timezone (don't change if already set to something reasonable)
CURRENT_TZ=$(timedatectl show --property=Timezone --value 2>/dev/null || cat /etc/timezone 2>/dev/null || echo "UTC")
if [ "${CURRENT_TZ}" = "Etc/UTC" ] || [ "${CURRENT_TZ}" = "UTC" ] || [ -z "${CURRENT_TZ}" ]; then
    # Server has default UTC — keep it (sensible default for servers)
    log_detail "Timezone: ${CURRENT_TZ} (UTC default, keeping as-is)"
else
    log_detail "Timezone: ${CURRENT_TZ} (preserving existing)"
fi

log_detail "System kernel parameters and limits tuned"

# =============================================================================
# STEP 24b: Configure Mail System (Postfix/Dovecot/OpenDKIM/Roundcube)
# =============================================================================
log_detail "Configuring mail system..."

# 1) Detect vmail UID/GID and update Postfix virtual_uid/gid_maps
VMAIL_UID=$(id -u vmail 2>/dev/null || echo "")
VMAIL_GID=$(id -g vmail 2>/dev/null || echo "")
if [ -n "${VMAIL_UID}" ] && [ -n "${VMAIL_GID}" ]; then
    POSTFIX_MAIN="${INSTALL_DIR}/etc/postfix/main.cf"
    if [ -f "${POSTFIX_MAIN}" ]; then
        sed -i "s|^virtual_uid_maps = static:.*|virtual_uid_maps = static:${VMAIL_UID}|" "${POSTFIX_MAIN}"
        sed -i "s|^virtual_gid_maps = static:.*|virtual_gid_maps = static:${VMAIL_GID}|" "${POSTFIX_MAIN}"
        log_detail "Postfix virtual UID/GID set to ${VMAIL_UID}:${VMAIL_GID} (vmail)"
    fi
fi

# 1b) Set Postfix myhostname/mydomain from server hostname
POSTFIX_MAIN="${INSTALL_DIR}/etc/postfix/main.cf"
if [ -f "${POSTFIX_MAIN}" ]; then
    _SRV_FQDN=$(hostname -f 2>/dev/null || hostname 2>/dev/null || echo "localhost")
    _SRV_DOMAIN=$(hostname -d 2>/dev/null || echo "")
    # If hostname -d returns empty, extract domain from FQDN — but only when the
    # FQDN has a recognisable letter-TLD suffix. Hostnames like "pn-test-ubuntu22.04"
    # contain a dot (the OS version) but are NOT FQDNs; naive substring parsing
    # produced mydomain="04" which Postfix accepted but broke mail routing.
    if [ -z "${_SRV_DOMAIN}" ] && echo "${_SRV_FQDN}" | grep -qE '\.[a-zA-Z]{2,}$'; then
        _SRV_DOMAIN=$(echo "${_SRV_FQDN}" | sed 's/^[^.]*\.//')
    fi
    # Defense in depth: if the derived domain still lacks a letter TLD, drop it.
    if ! echo "${_SRV_DOMAIN}" | grep -qE '\.[a-zA-Z]{2,}$'; then
        _SRV_DOMAIN=""
    fi
    # Fallback: panelica.local is a known-safe RFC 6761 reserved suffix
    # (better than the bare hostname, which Postfix 3.10+ may reject).
    [ -z "${_SRV_DOMAIN}" ] && _SRV_DOMAIN="panelica.local"
    # Normalize to lowercase — prevents PTR/SPF/DKIM/SSL mismatches
    _SRV_FQDN=$(echo "${_SRV_FQDN}" | tr 'A-Z' 'a-z')
    _SRV_DOMAIN=$(echo "${_SRV_DOMAIN}" | tr 'A-Z' 'a-z')

    # Postfix 3.10+ (Ubuntu 26.04) rejects IP addresses in myhostname — use safe placeholder
    if echo "${_SRV_DOMAIN}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'; then
        _SRV_DOMAIN="panelica.local"
        log_detail "Hostname is IP — Postfix using panelica.local placeholder (admin can change later)"
    fi

    sed -i "s|^myhostname = .*|myhostname = mail.${_SRV_DOMAIN}|" "${POSTFIX_MAIN}"
    sed -i "s|^mydomain = .*|mydomain = ${_SRV_DOMAIN}|" "${POSTFIX_MAIN}"
    log_detail "Postfix hostname set to mail.${_SRV_DOMAIN} (domain: ${_SRV_DOMAIN})"

    # smtp_helo_name: EHLO greeting must match PTR/MX for deliverability
    if ! grep -q "^smtp_helo_name" "${POSTFIX_MAIN}"; then
        echo "" >> "${POSTFIX_MAIN}"
        echo "# Mail deliverability - EHLO hostname (must match PTR record)" >> "${POSTFIX_MAIN}"
        echo "smtp_helo_name = mail.${_SRV_DOMAIN}" >> "${POSTFIX_MAIN}"
        log_detail "Postfix smtp_helo_name set to mail.${_SRV_DOMAIN}"
    else
        sed -i "s|^smtp_helo_name = .*|smtp_helo_name = mail.${_SRV_DOMAIN}|" "${POSTFIX_MAIN}"
    fi
fi

# 2) Ensure Postfix spool directories exist with correct ownership
for spool_dir in active bounce corrupt defer deferred flush hold incoming maildrop pid private public saved trace; do
    mkdir -p "${INSTALL_DIR}/var/spool/postfix/${spool_dir}"
done
chown -R postfix:postfix "${INSTALL_DIR}/var/spool/postfix"
chown postfix:postdrop "${INSTALL_DIR}/var/spool/postfix/maildrop" 2>/dev/null || true
chown postfix:postdrop "${INSTALL_DIR}/var/spool/postfix/public" 2>/dev/null || true
chmod 730 "${INSTALL_DIR}/var/spool/postfix/maildrop" 2>/dev/null || true
chmod 1730 "${INSTALL_DIR}/var/spool/postfix/public" 2>/dev/null || true

# Set postdrop binary setgid (required for sieve vacation sendmail submission)
chgrp postdrop "${INSTALL_DIR}/services/postfix/sbin/postdrop" 2>/dev/null || true
chmod g+s "${INSTALL_DIR}/services/postfix/sbin/postdrop" 2>/dev/null || true

# Create /usr/sbin/sendmail symlink (required by Dovecot sieve vacation)
ln -sf "${INSTALL_DIR}/services/postfix/sbin/sendmail" /usr/sbin/sendmail

# 3) Copy Postfix main.cf to system path (Postfix reads /etc/postfix/)
mkdir -p /etc/postfix
cp -f "${INSTALL_DIR}/etc/postfix/main.cf" /etc/postfix/main.cf 2>/dev/null || true
log_detail "Postfix main.cf synced to /etc/postfix/"

# 4) Ensure Postfix lookup tables exist (referenced by main.cf)
for _map in sender_access virtual_aliases virtual_domains virtual_mailboxes; do
    _mapfile="${INSTALL_DIR}/etc/postfix/${_map}"
    [ -f "$_mapfile" ] || touch "$_mapfile"
    ${INSTALL_DIR}/services/postfix/sbin/postmap hash:"$_mapfile" 2>/dev/null || true
done
# Also ensure /etc/aliases.db exists (Postfix logs warning without it)
[ -f /etc/aliases ] || echo "postmaster: root" > /etc/aliases
[ -f /etc/aliases.db ] || newaliases 2>/dev/null || postalias /etc/aliases 2>/dev/null || true

# 5) OpenDKIM setup
# RHEL 10+: opendkim-genkey is in opendkim-tools (EPEL 9) but may ship as
# opendkim-default-keygen in EPEL 10. Prefer opendkim-genkey, fall back.
DKIM_GENKEY=
if command -v opendkim-genkey >/dev/null 2>&1; then
    DKIM_GENKEY=opendkim-genkey
elif command -v opendkim-default-keygen >/dev/null 2>&1; then
    DKIM_GENKEY=opendkim-default-keygen
fi
if [ -n "${DKIM_GENKEY}" ]; then
    mkdir -p "${INSTALL_DIR}/etc/opendkim/keys"
    mkdir -p "${INSTALL_DIR}/var/run/opendkim"

    # Main config
    cat > "${INSTALL_DIR}/etc/opendkim/opendkim.conf" << 'DKIMCFG'
AutoRestart             Yes
AutoRestartRate         10/1h
Syslog                  yes
SyslogSuccess           yes
LogWhy                  yes
Canonicalization        relaxed/relaxed
ExternalIgnoreList      refile:/opt/panelica/etc/opendkim/TrustedHosts
InternalHosts           refile:/opt/panelica/etc/opendkim/TrustedHosts
KeyTable                refile:/opt/panelica/etc/opendkim/KeyTable
SigningTable            refile:/opt/panelica/etc/opendkim/SigningTable
Mode                    sv
PidFile                 /opt/panelica/var/run/opendkim/opendkim.pid
SignatureAlgorithm      rsa-sha256
UserID                  opendkim:opendkim
Socket                  inet:8891@localhost
DKIMCFG

    # TrustedHosts
    cat > "${INSTALL_DIR}/etc/opendkim/TrustedHosts" << 'TRUSTEOF'
127.0.0.1
localhost
::1
TRUSTEOF

    # Empty KeyTable and SigningTable (populated when domains enable DKIM)
    touch "${INSTALL_DIR}/etc/opendkim/KeyTable"
    touch "${INSTALL_DIR}/etc/opendkim/SigningTable"

    chown -R opendkim:opendkim "${INSTALL_DIR}/etc/opendkim/" 2>/dev/null || true
    chown opendkim:opendkim "${INSTALL_DIR}/var/run/opendkim" 2>/dev/null || true

    # Systemd override to use Panelica config
    mkdir -p /etc/systemd/system/opendkim.service.d
    cat > /etc/systemd/system/opendkim.service.d/panelica.conf << 'DKIMSVC'
[Service]
ExecStart=
ExecStart=/usr/sbin/opendkim -x /opt/panelica/etc/opendkim/opendkim.conf -P /opt/panelica/var/run/opendkim/opendkim.pid
PIDFile=/opt/panelica/var/run/opendkim/opendkim.pid
DKIMSVC

    log_detail "OpenDKIM configured (keys generated per-domain via panel)"
else
    log_warn "opendkim-genkey not found, DKIM signing will not be available"
fi

# 6) Randomize Roundcube des_key
# NOTE: sed is safe here — des_key is a single-line string value, not a multiline array
# (see builder.md DERS 5: Never use sed on PHP multiline arrays)
RC_CONFIG="${INSTALL_DIR}/services/roundcube/config/config.inc.php"
if [ -f "${RC_CONFIG}" ]; then
    RC_DES_KEY=$(openssl rand -hex 24)
    sed -i "s|\$config\['des_key'\] = '.*';|\$config['des_key'] = '${RC_DES_KEY}';|" "${RC_CONFIG}"
    log_detail "Roundcube des_key randomized"
fi

# 7) Dovecot log/run directories
mkdir -p "${INSTALL_DIR}/var/run/dovecot" "${INSTALL_DIR}/var/logs/dovecot"
chown -R dovecot:dovecot "${INSTALL_DIR}/var/run/dovecot"

# 8) Ensure Dovecot sieve plugin conf exists (for autoresponder vacation)
if [ ! -f "${INSTALL_DIR}/etc/dovecot/conf.d/90-sieve.conf" ]; then
    mkdir -p "${INSTALL_DIR}/etc/dovecot/conf.d"
    cat > "${INSTALL_DIR}/etc/dovecot/conf.d/90-sieve.conf" << 'SIEVE_CONF'
# Panelica Dovecot Sieve Configuration
# Vacation responses sent via submission_host (dovecot.conf)
# sieve_sendmail_path NOT used — Dovecot no_new_privs prevents setgid postdrop
protocol lmtp {
  mail_plugins = $mail_plugins sieve
}
plugin {
  sieve = file:~/sieve;active=~/.dovecot.sieve
  sieve_vacation_send_from_recipient = yes
  sieve_max_script_size = 1M
  sieve_max_actions = 32
  sieve_max_redirects = 4
}
SIEVE_CONF
    log_detail "Dovecot sieve config created"
else
    # Ensure sieve_vacation_send_from_recipient is set (prevents spam — uses real sender, not null <>)
    _SIEVE_CONF="${INSTALL_DIR}/etc/dovecot/conf.d/90-sieve.conf"
    if ! grep -q 'sieve_vacation_send_from_recipient' "$_SIEVE_CONF" 2>/dev/null; then
        sed -i '/sieve_max_script_size/i\  sieve_vacation_send_from_recipient = yes' "$_SIEVE_CONF"
        log_detail "Added sieve_vacation_send_from_recipient to existing sieve config"
    fi
fi

# 9) Ensure dovecot.conf has 3 critical lines (builder.md DERS 2)
# Without these, Sieve/autoresponder fails silently
_DOVECOT_CONF="${INSTALL_DIR}/etc/dovecot/dovecot.conf"
if [ -f "$_DOVECOT_CONF" ]; then
    # submission_host — sieve vacation sends via SMTP, not sendmail fork
    if ! grep -q 'submission_host' "$_DOVECOT_CONF"; then
        sed -i '/^ssl_min_protocol/a\\n# SUBMISSION HOST (for sieve vacation responses via SMTP)\nsubmission_host = localhost:25' "$_DOVECOT_CONF"
        log_detail "Added submission_host to dovecot.conf"
    fi
    # mail_plugins = quota — global plugin base
    if ! grep -q '^mail_plugins' "$_DOVECOT_CONF"; then
        sed -i '/^submission_host/a\\n# GLOBAL PLUGINS\nmail_plugins = quota' "$_DOVECOT_CONF"
        log_detail "Added mail_plugins to dovecot.conf"
    fi
    # !include conf.d/*.conf — load sieve/quota configs
    if ! grep -q '!include conf.d' "$_DOVECOT_CONF"; then
        echo -e "\n# INCLUDE CONF.D\n!include conf.d/*.conf" >> "$_DOVECOT_CONF"
        log_detail "Added !include conf.d to dovecot.conf"
    fi
fi

log_detail "Mail system configured"

# =============================================================================
# PRE-START: Verify shared library dependencies for all critical binaries
# =============================================================================
_check_missing_libs() {
    local missing_libs=""
    local checked=0
    for bin in \
        "${INSTALL_DIR}/services/postfix/sbin/postfix" \
        "${INSTALL_DIR}/services/dovecot/sbin/dovecot" \
        "${INSTALL_DIR}/services/nginx-customer/sbin/nginx" \
        "${INSTALL_DIR}/services/nginx-panel/sbin/nginx" \
        "${INSTALL_DIR}/services/mysql/bin/mysqld" \
        "${INSTALL_DIR}/services/postgresql/bin/postgres" \
        "${INSTALL_DIR}/services/proftpd/sbin/proftpd" \
        "${INSTALL_DIR}/services/bind/sbin/named" \
        "${INSTALL_DIR}/services/redis/bin/redis-server" \
        "${INSTALL_DIR}/services/clamav/sbin/clamd" \
        "${INSTALL_DIR}/services/apache/bin/httpd" \
        "${INSTALL_DIR}/services/php/8.4/bin/php" \
        "${INSTALL_DIR}/services/php/8.3/bin/php" \
        "${INSTALL_DIR}/services/php/8.2/bin/php" \
        "${INSTALL_DIR}/services/php/8.1/bin/php"; do
        [ -f "$bin" ] || continue
        checked=$((checked + 1))
        local libs=$(ldd "$bin" 2>/dev/null | grep "not found" | awk '{print $1}' | sort -u)
        [ -n "$libs" ] && missing_libs="${missing_libs}${bin}: ${libs}\n"
    done
    if [ -n "$missing_libs" ]; then
        log_warn "Missing shared libraries detected! Attempting auto-fix..."
        # Extract lib names and try to install matching packages
        local lib_names=$(echo -e "$missing_libs" | grep -oP '\S+\.so\.\d+' | sort -u)
        for lib in $lib_names; do
            if [ "${OS_FAMILY}" = "rhel" ]; then
                # dnf resolves the providing package straight from the soname
                log_detail "Attempting dnf install for ${lib}"
                dnf -y install "*/${lib}" >> "${LOG_FILE}" 2>&1 || true
            else
                local pkg=$(apt-cache search --names-only "^lib.*" 2>/dev/null | grep -i "${lib%%.*}" | head -1 | awk '{print $1}')
                if [ -n "$pkg" ]; then
                    log_detail "Installing ${pkg} (provides ${lib})"
                    apt-get install -y "$pkg" >> "${LOG_FILE}" 2>&1 || true
                fi
            fi
        done
        # Re-check after fix attempt
        local still_missing=""
        for bin in "${INSTALL_DIR}/services/"*/sbin/* "${INSTALL_DIR}/services/"*/bin/*; do
            [ -f "$bin" ] || continue
            local libs=$(ldd "$bin" 2>/dev/null | grep "not found" | awk '{print $1}' | sort -u)
            [ -n "$libs" ] && still_missing="${still_missing}  ${bin##*/}: ${libs}\n"
        done
        if [ -n "$still_missing" ]; then
            log_warn "Some libraries still missing (services may fail):\n${still_missing}"
        fi
    else
        log_detail "All ${checked} binaries have complete library dependencies"
    fi
    # Always succeed: a missing optional lib is a warning, not an install failure.
    # (Without this, the trailing `[ -n ... ] && ...` test returned 1 on success
    #  and the spinner cosmetically printed "failed".)
    return 0
}
run_with_spinner "Verifying shared library dependencies..." _check_missing_libs

# =============================================================================
# STEP 25: Configure nftables Firewall
# =============================================================================
log_step 25 ${TOTAL_STEPS} "Configuring nftables Firewall"

# Detect current SSH port to avoid locking ourselves out
# Extract the port (last colon-separated field of LocalAddress:Port).
# awk split avoids `rev`, which is absent on minimal Ubuntu 26.04 images.
CURRENT_SSH_PORT=$(ss -tlnp 2>/dev/null | grep sshd | head -1 | awk '{print $4}' | awk -F: '{print $NF}')
CURRENT_SSH_PORT=${CURRENT_SSH_PORT:-22}

cat > /etc/nftables.conf << NFTEOF
#!/usr/sbin/nft -f
flush ruleset

table inet filter {
    chain input {
        type filter hook input priority filter; policy drop;
        iif lo accept
        ct state established,related accept
        ip protocol icmp accept
        ip6 nexthdr ipv6-icmp accept
        tcp dport ${CURRENT_SSH_PORT} ct state new accept
        tcp dport { 80, 443 } ct state new accept
        tcp dport ${PANEL_PORT} ct state new accept
        tcp dport 53 ct state new accept
        udp dport 53 accept
        tcp dport 21 ct state new accept
        tcp dport 49152-65534 ct state new accept
        tcp dport 3002 ct state new accept
        tcp dport { 25, 465, 587, 110, 143, 993, 995 } ct state new accept
        limit rate 5/minute log prefix "nftables-drop: " drop
    }
    chain forward {
        type filter hook forward priority filter; policy drop;
        ct state established,related accept
        # Docker bridge traffic
        iifname "docker*" accept
        oifname "docker*" ct state new accept
        iifname "br-*" accept
        oifname "br-*" ct state new accept
    }
    chain output {
        type filter hook output priority filter; policy accept;
    }
}
NFTEOF

if [ "${OS_FAMILY}" = "rhel" ]; then
    # RHEL ships firewalld enabled; it owns the nftables ruleset and would flush
    # Panelica's rules. Disable it and use raw nftables (parity with Debian path).
    systemctl disable --now firewalld 2>/dev/null || true
    # RHEL's nftables.service loads /etc/sysconfig/nftables.conf — point it at ours.
    mkdir -p /etc/sysconfig
    echo 'include "/etc/nftables.conf"' > /etc/sysconfig/nftables.conf
    log_detail "Switched to nftables (Panelica's hardened firewall)"
fi
systemctl enable nftables 2>/dev/null || true
nft -f /etc/nftables.conf 2>/dev/null || log_warn "Failed to load nftables rules"
log_detail "nftables firewall configured (SSH port: ${CURRENT_SSH_PORT}, Docker forward rules included)"

# =============================================================================
# STEP 26: Start All Services (Ordered)
# =============================================================================
log_step 26 ${TOTAL_STEPS} "Starting Services"

start_service() {
    local name=$1
    local display_name=${2:-$1}
    systemctl start "panelica-${name}" 2>/dev/null
    if systemctl is-active --quiet "panelica-${name}" 2>/dev/null; then
        log_detail "${display_name} ${GREEN}running${NC}"
    else
        log_warn "${display_name} failed to start (check logs later)"
    fi
}

# Phase 1: Databases
echo -e "  ${ARROW} ${BOLD}Phase 1: Databases${NC}" | tee -a "${LOG_FILE}"
start_service redis "Redis"
# Fix Redis socket group for WordPress PHP-FPM access
sleep 1
if [ -S "${INSTALL_DIR}/var/run/redis.sock" ]; then
    chgrp panelica-web "${INSTALL_DIR}/var/run/redis.sock" 2>/dev/null
    chmod 770 "${INSTALL_DIR}/var/run/redis.sock" 2>/dev/null
fi
start_service postgresql "PostgreSQL"
sleep 2
start_service mysql "MySQL"
sleep 3

# MySQL socket symlink (after service starts)
ln -sf ${INSTALL_DIR}/var/run/mysqld/mysqld.sock ${INSTALL_DIR}/var/run/mysqld.sock 2>/dev/null || true

# Phase 2: PHP-FPM — start all five bundled versions (8.1-8.5).
# Earlier installs only enabled 8.3 + 8.4 by default and left 8.1/8.2/8.5
# in "ready, enable from Panel" state, but that produced inconsistent
# pn-service output across OS families (Debian's apt postinst path left
# 8.1/8.2 stopped while RHEL/Ubuntu sometimes ended up with them running
# anyway, depending on package install ordering). Customers expect all
# bundled PHP versions to be ready out of the box, so we just start the
# whole set here. Each FPM pool is lightweight when idle (no requests =
# no children), so the cost is negligible.
echo -e "  ${ARROW} ${BOLD}Phase 2: PHP Processing${NC}" | tee -a "${LOG_FILE}"
start_service "php-fpm-8.1" "PHP-FPM 8.1"
start_service "php-fpm-8.2" "PHP-FPM 8.2"
start_service "php-fpm-8.3" "PHP-FPM 8.3"
start_service "php-fpm-8.4" "PHP-FPM 8.4"
start_service "php-fpm-8.5" "PHP-FPM 8.5"

# Phase 3: Web Servers
echo -e "  ${ARROW} ${BOLD}Phase 3: Web Servers${NC}" | tee -a "${LOG_FILE}"
start_service apache "Apache"
start_service nginx-panel "Nginx Panel"
start_service nginx-customer "Nginx Customer"

# Phase 4: Backend Application
echo -e "  ${ARROW} ${BOLD}Phase 4: Backend Application${NC}" | tee -a "${LOG_FILE}"
start_service backend "Backend API"
sleep 3
start_service external-api "External API"

# Phase 5: DNS + FTP
echo -e "  ${ARROW} ${BOLD}Phase 5: DNS & FTP${NC}" | tee -a "${LOG_FILE}"
start_service bind "BIND DNS"
start_service proftpd "ProFTPD"

# Phase 6: Security
echo -e "  ${ARROW} ${BOLD}Phase 6: Security${NC}" | tee -a "${LOG_FILE}"
# Ensure auth.log exists for fail2ban sshd jail. On Debian 12+ rsyslog no
# longer creates /var/log/auth.log by default (journald is the system log),
# so the apt-installed fail2ban service crashes on first boot with "Have
# not found any log file for sshd jail" — failed state poisons systemctl
# --failed even though our own panelica-fail2ban is healthy. We touch the
# file, then stop+disable+MASK the system unit (mask blocks the apt
# postinst auto-start path) and reset-failed clears the prior crash.
touch /var/log/auth.log 2>/dev/null || true
chmod 640 /var/log/auth.log 2>/dev/null || true
chown root:adm /var/log/auth.log 2>/dev/null || chown root:root /var/log/auth.log 2>/dev/null || true
# Apt-installed system fail2ban auto-starts; stop+disable+mask so
# panelica-fail2ban (which uses our own jail.local + chroot logs) doesn't
# race "Server already running" and doesn't leave a failed unit behind.
systemctl stop fail2ban 2>/dev/null || true
systemctl disable fail2ban 2>/dev/null || true
systemctl mask fail2ban 2>/dev/null || true
systemctl reset-failed fail2ban 2>/dev/null || true
start_service fail2ban "Fail2ban"
# OpenDKIM — Postfix smtpd_milters expects it on port 8891; without this,
# all outgoing mail leaves unsigned (Authentication-Results: dkim=none) on
# every recipient (Gmail/Outlook/etc.). Debian's apt postinst auto-enables
# opendkim.service, RHEL's dnf does not — `enable --now` covers both.
systemctl enable --now opendkim 2>/dev/null && log_detail "OpenDKIM ${GREEN}running${NC} (signs outgoing mail per Panel → Email Authentication)" \
    || log_warn "OpenDKIM failed to start — outgoing mail will not be DKIM-signed"
# ClamAV/Freshclam disabled by default (uses ~1.5GB RAM)
# Users can enable via Panel Settings > Services > Enable at Boot
log_detail "ClamAV antivirus ${GREEN}ready${NC} (enable from Panel → Services to start scanning)"

# Phase 7: Background Daemons
echo -e "  ${ARROW} ${BOLD}Phase 7: Background Services${NC}" | tee -a "${LOG_FILE}"
start_service bandwidth "Bandwidth Monitor"
start_service bandwidth-limiter "eBPF Bandwidth Limiter"
start_service ftp-quota "FTP Quota Daemon"
start_service cron-scheduler "Cron Scheduler"

# Phase 8: Mail Services (Postfix/Dovecot) — disabled by default
# Users can enable via Panel Settings > Services > Enable at Boot
echo -e "  ${ARROW} ${BOLD}Phase 8: Mail Services${NC}" | tee -a "${LOG_FILE}"
log_detail "Postfix mail server ${GREEN}ready${NC} (enable from Panel → Services to start mail)"
log_detail "Dovecot IMAP/POP3 ${GREEN}ready${NC} (enable from Panel → Services to start mail)"

# =============================================================================
# STEP 27: Backend Health Check
# =============================================================================
log_step 27 ${TOTAL_STEPS} "Waiting for Backend to Complete Startup"

log_detail "Backend is running database migrations and seeding i18n data..."
log_detail "Running 530+ SQL migrations (typically 1-3 min, longer on slower hardware)..."

RETRIES=90
spin_chars='⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'
for i in $(seq 1 ${RETRIES}); do
    HTTP_CODE=$(curl -sk -o /dev/null -w "%{http_code}" https://localhost:3001/api/v1/setup/status 2>/dev/null || echo "000")
    if [ "${HTTP_CODE}" = "200" ]; then
        printf "\r%-70s\r" "" >&3
        log_info "Backend API is healthy and responding (HTTP 200)"
        break
    fi
    if [ $i -eq ${RETRIES} ]; then
        printf "\r%-70s\r" "" >&3
        log_warn "Backend did not respond after ${RETRIES} attempts. Check: ${INSTALL_DIR}/var/logs/backend/"
    fi
    sc="${spin_chars:$(( (i-1) % ${#spin_chars} )):1}"
    printf "\r    \033[0;36m%s\033[0m Running migrations & waiting for API... (%d/%d)" "$sc" "$i" "$RETRIES" >&3
    sleep 2
done
printf "\r%-70s\r" "" >&3

# Retry nginx-customer after backend is ready (may have failed on first attempt
# because default SSL cert or vhost configs were not yet available)
if ! systemctl is-active --quiet panelica-nginx-customer 2>/dev/null; then
    systemctl restart panelica-nginx-customer 2>/dev/null || true
    sleep 1
    if systemctl is-active --quiet panelica-nginx-customer 2>/dev/null; then
        log_detail "Nginx Customer recovered after backend startup"
    fi
fi

# Wait for the parallel locale import to finish before declaring the panel ready.
# Without this, install.sh shows "Complete" while the backend is still writing
# 30 language bundles into the DB → user opens dashboard, picks Turkish/Russian,
# nothing translates because the bundle still has stub data.
log_detail "Waiting briefly for translations (English first, then others in background)..."
# 60-second cap: English is always loaded first, so the panel is fully usable
# in English by the time this returns. Other 30 languages continue importing
# in the background — typically finish within 3-5 min depending on server speed.
SYNC_RETRIES=30   # 30 * 2s = 60 seconds
for i in $(seq 1 ${SYNC_RETRIES}); do
    SYNC_DONE=$(curl -sk https://localhost:3001/api/v1/public/i18n/sync/status 2>/dev/null | grep -oE '"completed":(true|false)' | head -1)
    if [ "${SYNC_DONE}" = '"completed":true' ]; then
        printf "\r%-70s\r" "" >&3
        log_info "All 31 languages ready"
        break
    fi
    if [ $i -eq ${SYNC_RETRIES} ]; then
        printf "\r%-70s\r" "" >&3
        log_info "Panel ready — English available now, you can start using it"
        log_detail "Other 30 languages continue loading in background (~3-5 min depending on server speed)"
    fi
    sc="${spin_chars:$(( (i-1) % ${#spin_chars} )):1}"
    printf "\r    \033[0;36m%s\033[0m Loading translations... (%d/%d)" "$sc" "$i" "$SYNC_RETRIES" >&3
    sleep 2
done
printf "\r%-70s\r" "" >&3

# =============================================================================
# STEP 28: Installation Complete
# =============================================================================
log_step 28 ${TOTAL_STEPS} "Finalizing Installation"

# Service status summary (only check services that should be running)
RUNNING=0
FAILED=0
TOTAL=0
FAILED_LIST=""
for svc in redis postgresql mysql nginx-panel nginx-customer apache \
           php-fpm-8.3 php-fpm-8.4 \
           bind proftpd fail2ban \
           backend external-api bandwidth bandwidth-limiter ftp-quota \
           cron-scheduler; do
    TOTAL=$((TOTAL + 1))
    if systemctl is-active --quiet "panelica-${svc}" 2>/dev/null; then
        RUNNING=$((RUNNING + 1))
    else
        FAILED=$((FAILED + 1))
        FAILED_LIST="${FAILED_LIST}    ${CROSSMARK} panelica-${svc}\n"
    fi
done

INSTALL_TIME=$SECONDS
MINUTES=$((INSTALL_TIME / 60))
SECS=$((INSTALL_TIME % 60))

echo ""
echo -e "${CYAN}"
cat << 'DONEBANNER'
    ╔═══════════════════════════════════════════════════════════╗
    ║                                                           ║
    ║         Installation Complete Successfully!               ║
    ║                                                           ║
    ╚═══════════════════════════════════════════════════════════╝
DONEBANNER
echo -e "${NC}"

echo -e "    ${BOLD}Service Status:${NC}  ${GREEN}${RUNNING}${NC}/${TOTAL} services running"
if [ -n "${FAILED_LIST}" ]; then
    echo -e "${FAILED_LIST}"
fi
echo -e "    ${BOLD}Optional Services:${NC} PHP 8.1/8.2/8.5, Postfix, Dovecot, ClamAV, pgAdmin4"
echo -e "    ${DIM}Enable from Panel Settings > Services when needed${NC}"

echo ""
echo -e "    ${BOLD}Access Your Panel:${NC}"
echo -e "    ${ARROW} https://${SERVER_IP}:${PANEL_PORT}"
if [ "${SERVER_HOSTNAME}" != "${SERVER_IP}" ]; then
    echo -e "    ${ARROW} https://${SERVER_HOSTNAME}:${PANEL_PORT}"
fi
echo ""
echo -e "    ${BOLD}Next Steps:${NC}"
if [ "${SERVER_HOSTNAME}" != "${SERVER_IP}" ]; then
    echo -e "    ${DIM}1.${NC} Open ${BOLD}https://${SERVER_HOSTNAME}:${PANEL_PORT}${NC} or ${BOLD}https://${SERVER_IP}:${PANEL_PORT}${NC} in your browser"
else
    echo -e "    ${DIM}1.${NC} Open ${BOLD}https://${SERVER_IP}:${PANEL_PORT}${NC} in your browser"
fi
echo -e "    ${DIM}2.${NC} Complete the Setup Wizard (set root password)"
echo -e "    ${DIM}3.${NC} Enter your license key or start a trial"
echo -e "    ${DIM}4.${NC} You're ready to go!"
echo ""
echo -e "    ${CYAN}╭─────────────────────────────────────────────╮${NC}"
echo -e "    ${CYAN}│${NC}  ${BOLD}Default Login:${NC}  Username: ${GREEN}${BOLD}root${NC}              ${CYAN}│${NC}"
echo -e "    ${CYAN}╰─────────────────────────────────────────────╯${NC}"
echo ""
echo -e "    ${DIM}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "    ${DIM}Install time:  ${MINUTES}m ${SECS}s${NC}"
echo -e "    ${DIM}Install log:   ${LOG_FILE}${NC}"
echo -e "    ${DIM}Config file:   ${INSTALL_DIR}/panelica.conf${NC}"
echo -e "    ${DIM}Services:      ${INSTALL_DIR}/bin/pn-service status all${NC}"
echo ""
