#!/bin/bash

# ─────────────────────────────────────────────────────────────────────────────
# Production Docker Host Setup & Hardening Script — v3.1.0
# For Ubuntu Server 24.04 LTS (Noble) and 26.04 LTS (Resolute)
# Suitable for both Kamal 2.x deployment hosts and remote builder hosts
#
# Part of RailsFast (https://railsfast.com) — free & open source, MIT licensed.
#
# Usage (as root, on a fresh server):
#   wget -qO railsfast-setup.sh https://setup.railsfast.com
#   less railsfast-setup.sh    # always read a script before you run it
#   bash railsfast-setup.sh
#
# It is also safe to run:
#   - unattended via cloud-init (no prompts, ever — see the RailsFast docs)
#   - again on a server that was set up with v2 or v3 (idempotent: it upgrades
#     the config in place and removes v2 footguns — see changelog below)
#
# Optional knobs (env vars, e.g. `INSTALL_MOSH=true bash railsfast-setup.sh`):
#   DOCKER_USERNAME=docker  # the user Kamal deploys as (ssh.user in deploy.yml)
#   SSH_PUBLIC_KEY="ssh-ed25519 AAAA..."  # explicit deploy key (else copied
#                           # from root's / the sudo user's authorized_keys)
#   INSTALL_MOSH=true       # install mosh + open UDP 60000:61000
#   INSTALL_AUDITD=true     # auditd with Docker-config audit rules
#   SETUP_SWAP=auto         # auto = add swap unless the server already has
#                           # some (2G; 4G on >=4GB-RAM servers); true; false
#   SKIP_OS_CHECK=true      # run on untested Ubuntu versions (at your own risk)
#   ALLOW_INCOMPLETE_SSH=true  # finish (with warnings, and WITHOUT the
#                           # canonical success line) even when no usable SSH
#                           # key exists. Default: a missing deploy key FAILS
#                           # verification — Kamal couldn't connect anyway.
#
# Maintenance notes for future versions of this script:
#   - Inline comments cite exact sources (docs / source code / manpages) for
#     every non-obvious decision. Please keep them updated when changing lines.
#   - Output contracts other systems depend on — do not break them:
#       1. The final success line must contain "SUCCESS: Setup complete!"
#          (the RailsFast quickstart tells users to wait for that exact text,
#          and cloud-init reboots after it) — and it must print ONLY when
#          verification passed. On any failure the script exits non-zero,
#          cloud-init does NOT reboot, and the log shows what failed.
#       2. The script must never prompt (cloud-init runs it unattended).
#
# Changelog v3.1.0:
#   - journald is now persistent and sized for APP logs, not just host logs.
#     Kamal deploys prune old containers, and a container's json-file log dies
#     with it — so per-container logs don't survive a deploy. RailsFast's
#     config/deploy.yml now uses the journald Docker log driver, routing app
#     logs into the host journal (which outlives containers). To back that:
#       * the journald drop-in now sets Storage=persistent, guaranteeing
#         /var/log/journal exists so logs also survive reboots (the stock
#         Storage=auto is persistent only if that dir already exists), and
#       * SystemMaxUse is raised 500M -> 5G, since app logs now live here.
#     Re-running on a v3.0.x server upgrades the drop-in in place. daemon.json
#     stays json-file (host default) so non-journald containers still rotate.
#
# Changelog v3.0.1:
#   - Swap on EVERY server by default (was: only <=2GB-RAM boxes). Rationale:
#     Kamal's gapless deploys briefly run the OLD and NEW app containers side
#     by side, so the memory spike that invites the kernel OOM killer (whose
#     favorite target is Postgres) recurs on every deploy, on every box size.
#     Tiered 2G (<4GB RAM) / 4G (>=4GB RAM), HARD-CAPPED at 4G: a bounded
#     buffer absorbs real spikes and lets pathological states (leaks) still
#     terminate; oversized swap just prolongs a degraded "molasses" state
#     where the site crawls for hours instead of restarting in seconds.
#     swappiness stays 10 (emergency valve, not active memory). Existing v2 /
#     v3.0.0 servers gain swap the next time the script is re-run on them.
#
# Changelog v2 -> v3:
#   - REMOVED the daily `docker system prune -af --volumes` cron. It deleted the
#     stopped containers and images Kamal retains for `kamal rollback`. Kamal
#     already prunes after every successful deploy, keeping the last 5 versions.
#     Re-running this script on a v2 server deletes the old cron job.
#   - REMOVED aide, rkhunter and logwatch. Without a configured mail path their
#     reports go nowhere, and they dragged in postfix (logwatch hard-depends on
#     a mail server), a full Ruby (rkhunter recommends unhide.rb), and other
#     cruft. All apt installs now use --no-install-recommends.
#     (Want them? Install them deliberately, with mail delivery configured.)
#   - Docker is now installed from Docker's official apt repository instead of
#     the get.docker.com convenience script (which Docker does not recommend
#     for production systems).
#   - SSH hardening moved from overwriting /etc/ssh/sshd_config to a drop-in at
#     /etc/ssh/sshd_config.d/00-railsfast.conf. sshd uses the FIRST value it
#     reads for each keyword and Ubuntu includes sshd_config.d/*.conf before
#     everything else, so v2's settings could be silently overridden by
#     cloud-init's 50-cloud-init.conf (e.g. PasswordAuthentication yes on
#     password-provisioned servers). The drop-in wins over everything, and the
#     effective config is now verified with `sshd -T` instead of grepping files.
#   - Password login is only disabled when an SSH key is actually installed for
#     the deploy user — no more lockout footgun on password-provisioned servers.
#   - REMOVED the pinned KexAlgorithms/Ciphers/MACs lists (stale: they silently
#     disabled OpenSSH's newer post-quantum key exchange). Ubuntu's defaults are
#     stronger and maintained. Re-running on a v2 server removes the old pins.
#   - REMOVED `AddressFamily inet` (it made sshd unreachable on IPv6-only VPSes)
#     and the deprecated `ChallengeResponseAuthentication` alias.
#   - REMOVED the logrotate rule for Docker's json log files (Docker manages
#     them itself via daemon.json; external rotation with copytruncate can
#     corrupt them) and the fail2ban "docker" jail (its regex matched nothing).
#   - No more `ufw --force reset` (it wiped rules users added themselves).
#   - unattended-upgrades: distro config is left intact; our settings live in a
#     drop-in. Security updates stay enabled by default.
#   - NEW hardening: GatewayPorts pinned off, IPv6 ICMP redirects disabled,
#     kexec disabled, fail2ban incremental bans, fail2ban reads the systemd
#     journal (with the journal match Ubuntu 25.10+ ships as a bug fix).
#   - NEW: creates a 2G swap file on small (<= 2GB RAM) servers, caps journald
#     disk usage, verifies Docker access as the deploy user, and prints honest
#     next steps (including that Docker-published ports bypass ufw and that a
#     provider-level firewall is recommended).
#   - Release-review fixes on top of the above:
#       * Verification is FATAL: success line only on a fully verified setup;
#         non-zero exit (and no cloud-init reboot) otherwise.
#       * ufw opens the EFFECTIVE sshd port(s) read from `sshd -T`, and
#         fail2ban watches the same ports — servers on custom SSH ports no
#         longer lock themselves out.
#       * Deploy-key seeding copies only unrestricted key lines (cloud images
#         plant root keys with command="..." prefixes that would break Kamal)
#         and validates the result with ssh-keygen before password auth is
#         ever disabled. New SSH_PUBLIC_KEY env var for explicit provisioning.
#       * Conflicting distro container packages (docker.io, podman-docker,
#         containerd, runc, ...) are removed first, per Docker's install docs.
#       * SSH drop-in changes are transactional (previous drop-in restored if
#         the new one fails validation); an effective-config mismatch after
#         reload is now a hard failure.
#       * fail2ban config lives in jail.d/railsfast.conf and auto-upgrade
#         prefs in 21auto-upgrades-railsfast — your own jail.local /
#         20auto-upgrades are never touched (fail2ban reads jail.conf ->
#         jail.d/*.conf -> jail.local, LAST value wins, so your jail.local
#         still overrides us).
#       * Dropped the fs.file-max and tcp_max_syn_backlog pins: modern
#         kernels scale both with RAM, so fixed values could LOWER them.
#   - Second review round:
#       * Key files containing ONLY restricted/forced-command keys (as copied
#         wholesale by v2) no longer count as usable; SSH_PUBLIC_KEY is
#         ensured present even when a file already exists.
#       * A missing deploy key now FAILS the run by default (Kamal couldn't
#         connect anyway); ALLOW_INCOMPLETE_SSH=true is the explicit opt-out,
#         and even then the canonical success line is withheld.
#       * The sshd drop-in transaction now spans validation, reload AND
#         effective-config verification — rollback + reload on any failure.
#       * IP detection can't abort under `set -e -o pipefail` on IPv6-only
#         hosts (falls back IPv4 route -> IPv6 route -> hostname -I).
#       * Version marker is written next to the final verdict; cleanup moved
#         before verification; `apt autoremove` dropped (surprise removals on
#         repurposed boxes; unattended-upgrades handles the ongoing case).
#       * fail2ban backend scoped to [sshd] (a global backend=systemd breaks
#         file-based jails admins add); v2's [docker] jail is removed
#         surgically so admin jails in the same file survive.
#       * `systemctl enable --now docker` (a stopped daemon with an unchanged
#         config was never started before).
# ─────────────────────────────────────────────────────────────────────────────

# -E so the ERR trap fires inside functions/subshells too; -u catches typoed
# variables; pipefail catches failures upstream of a pipe.
set -Eeuo pipefail
IFS=$'\n\t'

# --- Constants ---
SCRIPT_VERSION="3.1.0"
SUPPORTED_UBUNTU_VERSIONS=("24.04" "26.04")
# 24.04 (Noble) and 26.04 (Resolute Raccoon, released 2026-04-23, LTS to 2031)
# are the Ubuntu LTS releases Docker publishes apt dists for (both verified at
# https://download.docker.com/linux/ubuntu/dists/ — non-LTS interims come and
# go there; only list LTS releases here).
# Ubuntu 26.04 facts: https://documentation.ubuntu.com/release-notes/26.04/
MIN_RAM_MB=1024
RECOMMENDED_RAM_MB=1900   # a "2GB" VPS reports ~1900-1970MB usable to free(1)
MIN_DISK_GB=20            # checked against AVAILABLE space on /, not disk size
VERSION_MARKER="/etc/railsfast-setup.version"

# --- Optional knobs (env-overridable) ---
DOCKER_USERNAME="${DOCKER_USERNAME:-docker}"
SSH_PUBLIC_KEY="${SSH_PUBLIC_KEY:-}"
INSTALL_MOSH="${INSTALL_MOSH:-false}"
INSTALL_AUDITD="${INSTALL_AUDITD:-false}"
SETUP_SWAP="${SETUP_SWAP:-auto}"
SKIP_OS_CHECK="${SKIP_OS_CHECK:-false}"
ALLOW_INCOMPLETE_SSH="${ALLOW_INCOMPLETE_SSH:-false}"

# --- Non-interactive everywhere (cloud-init safe) ---
# DEBIAN_FRONTEND stops debconf prompts; NEEDRESTART_MODE=a makes Ubuntu's
# needrestart auto-restart services instead of drawing a TUI dialog when this
# script is run manually over SSH.
export DEBIAN_FRONTEND=noninteractive
export NEEDRESTART_MODE=a

# --- Aesthetics ---
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
ALIEN='\xF0\x9F\x91\xBD'
NC='\033[0m'

# --- Helpers ---
print_message() {
    local color=$1
    local message=$2
    echo -e "${color}${ALIEN} ${message}${NC}"
}

print_error()   { print_message "${RED}" "ERROR: $1"; }
print_warning() { print_message "${YELLOW}" "WARNING: $1"; }
print_success() { print_message "${GREEN}" "SUCCESS: $1"; }

# apt-get that never prompts and waits for competing apt locks instead of
# failing. On first boot, cloud-init's own apt steps and the apt-daily /
# unattended-upgrades timers can hold the dpkg lock — DPkg::Lock::Timeout
# makes apt wait (up to 10 min) rather than die with "could not get lock".
# --force-confdef/confold keep upgraded packages' existing config files, so
# `apt-get upgrade` can never stop to ask a conffile question mid-run.
apt_get() {
    apt-get \
        -o DPkg::Lock::Timeout=600 \
        -o Dpkg::Options::=--force-confdef \
        -o Dpkg::Options::=--force-confold \
        "$@"
}

handle_error() {
    local line_number=$1
    print_error "Script failed on line ${line_number}"
    print_error "Please check the logs above for more information"
    exit 1
}

trap 'handle_error ${LINENO}' ERR

# --- Pre-flight checks ---
check_root() {
    if [[ $EUID -ne 0 ]]; then
        print_error "This script must be run as root"
        exit 1
    fi
}

check_os() {
    # /etc/os-release is guaranteed present on every Ubuntu flavor (including
    # minimal cloud images, which do NOT always ship lsb_release — v2 used
    # lsb_release and would false-negative there).
    if [[ ! -r /etc/os-release ]]; then
        print_error "/etc/os-release not found. Is this Ubuntu?"
        exit 1
    fi

    # shellcheck disable=SC1091
    . /etc/os-release

    if [[ "${ID:-}" != "ubuntu" ]]; then
        print_error "This script requires Ubuntu (found ${ID:-unknown})"
        exit 1
    fi

    local supported=false
    for version in "${SUPPORTED_UBUNTU_VERSIONS[@]}"; do
        [[ "${VERSION_ID:-}" == "$version" ]] && supported=true
    done

    if [[ "$supported" != "true" ]]; then
        # NOTE: IFS is \n\t globally, so join the array with spaces explicitly
        # (a bare "${arr[*]}" would join with a newline and mangle the message).
        local supported_list
        supported_list=$(IFS=' '; echo "${SUPPORTED_UBUNTU_VERSIONS[*]}")
        if [[ "$SKIP_OS_CHECK" == "true" ]]; then
            print_warning "Ubuntu ${VERSION_ID:-unknown} is untested (supported: ${supported_list}). Continuing because SKIP_OS_CHECK=true."
        else
            print_error "This script supports Ubuntu ${supported_list} (found ${VERSION_ID:-unknown})"
            print_error "Set SKIP_OS_CHECK=true to run anyway, at your own risk"
            exit 1
        fi
    fi

    # Used for the Docker apt repo (dists are named by codename: noble,
    # resolute, ...). Docker's install docs read the same field.
    UBUNTU_CODENAME_DETECTED="${VERSION_CODENAME:-}"
    if [[ -z "$UBUNTU_CODENAME_DETECTED" ]]; then
        print_error "Could not detect the Ubuntu codename from /etc/os-release"
        exit 1
    fi
}

check_resources() {
    local total_ram_mb
    local avail_disk_gb
    total_ram_mb=$(free -m | awk '/^Mem:/{print $2}')
    avail_disk_gb=$(df -BG / | awk 'NR==2 {print $4}' | sed 's/G//')

    if (( total_ram_mb < MIN_RAM_MB )); then
        print_error "Insufficient RAM. Required: ${MIN_RAM_MB}MB, Found: ${total_ram_mb}MB"
        exit 1
    fi

    if (( total_ram_mb < RECOMMENDED_RAM_MB )); then
        print_warning "Only ${total_ram_mb}MB RAM. 2GB+ is recommended to run Rails + Postgres comfortably."
    fi

    if (( avail_disk_gb < MIN_DISK_GB )); then
        print_error "Insufficient free disk space. Required: ${MIN_DISK_GB}GB, Available: ${avail_disk_gb}GB"
        exit 1
    fi

    TOTAL_RAM_MB="$total_ram_mb"
}

detect_previous_setup() {
    if [[ -f "$VERSION_MARKER" ]]; then
        print_message "${YELLOW}" "Server was previously set up with version $(cat "$VERSION_MARKER"). Re-running in update mode."
    elif [[ -f /etc/cron.daily/docker-cleanup ]]; then
        # v2 didn't write a version marker; its prune cron is its fingerprint.
        print_message "${YELLOW}" "Server was previously set up with v2 of this script. Upgrading in place."
    fi
}

print_message "${YELLOW}" "RailsFast production server setup v${SCRIPT_VERSION}"
print_message "${YELLOW}" "Performing pre-flight checks..."
check_root
check_os
check_resources
detect_previous_setup

# --- System updates ---
print_message "${YELLOW}" "Updating system packages..."
apt_get update
apt_get upgrade -y

# --- Essential packages ---
# --no-install-recommends keeps "recommended" baggage off the box — apt(8):
# "Do not consider recommended packages as a dependency for installing."
# https://manpages.ubuntu.com/manpages/noble/man8/apt-get.8.html
# CAVEAT it only suppresses Recommends, not Depends: that's why v2's logwatch
# always pulled postfix (logwatch DEPENDS on default-mta | mail-transport-agent,
# https://packages.ubuntu.com/noble/logwatch) and why removing logwatch/
# rkhunter/aide (not just flipping this flag) was the real fix for the
# postfix/ruby/unhide package bloat users reported.
print_message "${YELLOW}" "Installing essential packages..."
# - adduser: used below for the deploy user; Ubuntu 26.04 minimal images no
#   longer ship it in the base system (verified in the ubuntu:26.04 image).
# - python3-systemd: fail2ban's systemd-journal backend. Technically redundant
#   (fail2ban hard-Depends on it: https://sources.debian.org/data/main/f/fail2ban/1.1.0-8/debian/control)
#   but kept explicit as documentation of the requirement.
# - apparmor: present by default on Ubuntu, listed to guarantee it on minimal
#   images. Docker applies its apparmor profile to containers automatically
#   when AppArmor is active.
# - sysstat: sar metrics ("why was the server slow at 3am?"). Collection is
#   OFF by default on 24.04 and enabled just below.
apt_get install -y --no-install-recommends \
    adduser \
    ca-certificates \
    curl \
    ufw \
    fail2ban \
    python3-systemd \
    unattended-upgrades \
    apparmor \
    sysstat

if [[ "$INSTALL_MOSH" == "true" ]]; then
    # Mosh authenticates over SSH, then talks UDP 60000-61000 ("Mosh will log
    # the user in via SSH, then start a connection on a UDP port between
    # 60000 and 61000." — https://mosh.org/). The matching ufw rule is added
    # in the firewall section below.
    print_message "${YELLOW}" "Installing mosh (INSTALL_MOSH=true)..."
    apt_get install -y --no-install-recommends mosh
fi

# Enable sysstat data collection. On 24.04 the package installs with
# ENABLED="false" (https://wiki.debian.org/sysstat); 26.04 ships it enabled,
# where this sed is a harmless no-op.
if [[ -f /etc/default/sysstat ]]; then
    sed -i 's/^ENABLED="false"/ENABLED="true"/' /etc/default/sysstat
    systemctl enable --now sysstat >/dev/null 2>&1 || true
fi

# --- Swap (every server) ---
# Kamal's gapless deploys briefly run the old AND new app containers side by
# side (plus image extraction), so the memory spike that invites the OOM
# killer recurs on EVERY deploy, on every box size — and the OOM killer's
# favorite target on these hosts is Postgres. A bounded swap buffer absorbs
# that spike; with swappiness=10 (set below) it sits untouched otherwise.
# Deliberately HARD-CAPPED at 4G: the spike is per-deploy, not proportional
# to RAM, and oversized swap converts "fast OOM + container restart" into
# hours of degraded, actively-swapping "molasses mode" — worse for users.
# Skipped when the box already has swap (so re-runs and pre-swapped images
# are no-ops).
setup_swap() {
    if [[ "$SETUP_SWAP" == "false" ]]; then
        return 0
    fi
    if swapon --show --noheadings | grep -q .; then
        print_message "${GREEN}" "Swap already configured. Skipping."
        return 0
    fi

    # Tiered size: bigger boxes tend to run several apps (more concurrent
    # deploy spikes). "4GB" VPSes report ~3800-3950MB to free(1), 2GB ones
    # ~1900MB — 3500 splits the tiers cleanly.
    local swap_size="2G"
    (( TOTAL_RAM_MB >= 3500 )) && swap_size="4G"

    # Swap files on btrfs/zfs need special handling (NOCOW/zvols), so only
    # auto-create on ext4/xfs — which is what every major VPS image uses.
    local root_fstype
    root_fstype=$(findmnt -no FSTYPE / || echo "unknown")
    if [[ "$root_fstype" != "ext4" && "$root_fstype" != "xfs" ]]; then
        print_warning "Root filesystem is ${root_fstype}; skipping automatic swap file creation."
        return 0
    fi

    print_message "${YELLOW}" "Creating ${swap_size} swap file (RAM: ${TOTAL_RAM_MB}MB)..."
    if ! fallocate -l "$swap_size" /swapfile 2>/dev/null; then
        print_warning "Could not allocate a swap file. Skipping (not fatal)."
        rm -f /swapfile
        return 0
    fi
    chmod 600 /swapfile
    # Swap is a nice-to-have: no step here may abort the whole setup.
    if ! mkswap /swapfile >/dev/null 2>&1 || ! swapon /swapfile 2>/dev/null; then
        print_warning "Could not activate the swap file. Removing it and continuing (not fatal)."
        swapoff /swapfile 2>/dev/null || true
        rm -f /swapfile
        return 0
    fi
    if ! grep -q '^/swapfile' /etc/fstab; then
        echo '/swapfile none swap sw 0 0' >> /etc/fstab
    fi
    # swappiness 10: keep swap as a pressure valve, not general-purpose memory
    # — Postgres and Puma latency suffer badly when actively swapping.
    echo 'vm.swappiness = 10' > /etc/sysctl.d/99-railsfast-swap.conf
    sysctl -p /etc/sysctl.d/99-railsfast-swap.conf >/dev/null 2>&1 || true
}
setup_swap

# --- Time synchronization ---
# Accurate clocks matter for TLS, Let's Encrypt issuance (kamal-proxy) and
# log forensics. 24.04 ships systemd-timesyncd; 25.10+/26.04 ship chrony as
# the default time daemon ("Chrony ... comes pre-installed as the new default
# time-daemon in Ubuntu 25.10, replacing systemd-timesyncd" —
# https://documentation.ubuntu.com/release-notes/25.10/ ; also
# https://ubuntu.com/server/docs/how-to/networking/timedatectl-and-timesyncd/).
# v2-era servers installed chrony explicitly. Either daemon is plenty for a
# web host: keep whichever is running, install timesyncd only when neither is.
# (Installing chrony alongside timesyncd conflicts — timedatectl "steps back"
# when chrony is present — so we never install both.)
print_message "${YELLOW}" "Configuring time synchronization..."
if systemctl is-active --quiet chrony 2>/dev/null; then
    print_message "${GREEN}" "chrony already active. Keeping it."
else
    apt_get install -y --no-install-recommends systemd-timesyncd
    systemctl enable --now systemd-timesyncd
    timedatectl set-ntp true
fi

# --- AppArmor ---
# Enabled by default on Ubuntu; asserted here for minimal images. While
# AppArmor is active, Docker confines every container with its docker-default
# profile automatically (https://docs.docker.com/engine/security/apparmor/).
systemctl enable --now apparmor

# --- Kernel parameters ---
print_message "${YELLOW}" "Configuring kernel security parameters..."
# Written to the SAME path v2 used (/etc/sysctl.d/99-security.conf) so re-runs
# and v2 upgrades replace the file instead of stacking a second config.
#
# Two v2 lines are deliberately GONE:
#   - kernel.pid_max = 65536: it LOWERED the limit (systemd defaults it to
#     4194304 via /usr/lib/sysctl.d) for no benefit.
#   - net.ipv4.ip_local_port_range = 1024 65000: widening the ephemeral range
#     down to 1024 lets outbound connections squat on low ports — including
#     5555, where Kamal's local-registry SSH tunnel must bind on this server
#     (see the SSH section). Rare, but it randomly breaks deploys. The kernel
#     default (32768-60999) is plenty for a single-app host.
#   - net.ipv6.conf.*.accept_ra is deliberately NOT set to 0: several clouds
#     configure IPv6 via router advertisements, and disabling RA breaks their
#     IPv6 entirely. Cloud vswitches filter rogue RAs anyway.
#   - fs.file-max and net.ipv4.tcp_max_syn_backlog (both pinned in v2) are
#     gone too: modern kernels scale both with available RAM (see
#     https://www.kernel.org/doc/Documentation/networking/ip-sysctl.rst —
#     "it will increase in proportion to the memory of machine"), so a fixed
#     pin can LOWER them on larger servers. Per-process fd limits — the ones
#     that actually bite Rails/Puma — are set via Docker default-ulimits.
cat <<'EOF' > /etc/sysctl.d/99-security.conf
# ICMP redirects: a MITM vector on shared L2 segments; a VPS is not a router,
# so never send or accept them (IPv4 AND IPv6 — v2 only covered IPv4).
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv6.conf.all.accept_redirects = 0
net.ipv6.conf.default.accept_redirects = 0

# Source-address sanity: drop spoofed/martian packets at the door.
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv6.conf.all.accept_source_route = 0
net.ipv6.conf.default.accept_source_route = 0

# ICMP noise hygiene.
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_ignore_bogus_error_responses = 1

# SYN-flood resilience for a host that terminates 80/443 for the internet.
# (No tcp_max_syn_backlog pin: the kernel sizes it from RAM, and with
# syncookies enabled a full backlog degrades gracefully anyway.)
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 5

# Docker routes traffic between the kamal bridge network and the world.
# Without this, published ports and container egress break.
net.ipv4.ip_forward = 1

# Headroom for a busy container host (raises the 65530 default; harmless).
vm.max_map_count = 262144

# Hide kernel internals from unprivileged eyes (kernel pointer leaks, dmesg,
# perf side channels) and disable unprivileged eBPF — none of which a Rails +
# Docker workload needs.
kernel.kptr_restrict = 2
kernel.dmesg_restrict = 1
kernel.perf_event_paranoid = 3
kernel.unprivileged_bpf_disabled = 1
net.core.bpf_jit_harden = 2

# ptrace_scope 2: only processes with CAP_SYS_PTRACE may ptrace. Debuggers
# inside containers are unaffected for their own children; the Ruby debug gem
# uses sockets, not ptrace.
kernel.yama.ptrace_scope = 2

# Block runtime kernel replacement via kexec (one-way until reboot).
# CAVEAT: if you ever set up kdump crash dumps, remove this line first.
kernel.kexec_load_disabled = 1

# Filesystem link/dump hardening.
fs.protected_hardlinks = 1
fs.protected_symlinks = 1
fs.suid_dumpable = 0

# Martian logging off: on an internet-exposed host it just churns the journal
# on small disks; enable temporarily (=1) when debugging weird routing.
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.default.log_martians = 0
EOF

sysctl --system >/dev/null

# --- Docker installation (official apt repository) ---
# Docker's convenience script (get.docker.com, which v2 used) "isn't
# recommended for production environments"; the apt repo is the supported
# path and gives us signed, pinnable, unattended-upgradeable packages.
# Install steps and package list follow the official docs verbatim:
# https://docs.docker.com/engine/install/ubuntu/
# Supported Ubuntu versions there (checked 2026-07): 26.04 LTS (resolute),
# 25.10 (questing), 24.04 LTS (noble), 22.04 LTS (jammy).
print_message "${YELLOW}" "Installing Docker from Docker's official apt repository..."

# Remove conflicting container packages first, exactly the list Docker's
# docs say to uninstall ("Uninstall old versions",
# https://docs.docker.com/engine/install/ubuntu/). Fresh servers have none of
# these; this protects re-purposed boxes where someone once installed
# Ubuntu's docker.io/podman stack (its containerd/runc conflict with
# containerd.io). `remove`, not `purge`: per the same docs, images/volumes
# under /var/lib/docker are preserved. NOTE: if distro-docker containers are
# RUNNING they will stop — unavoidable; docker-ce can't coexist with them.
for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do
    if dpkg -l "$pkg" 2>/dev/null | grep -q '^ii'; then
        print_warning "Removing conflicting package '${pkg}' (replaced by Docker CE packages)"
        apt_get remove -y "$pkg"
    fi
done

install -m 0755 -d /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
chmod a+r /etc/apt/keyrings/docker.asc

# One canonical repo definition in the Deb822 format current docs use.
# (Docker's docs switched from the one-line docker.list to docker.sources;
# we remove the legacy file so upgrades from v2 / get.docker.com don't leave
# a duplicate repo behind.)
rm -f /etc/apt/sources.list.d/docker.list
cat <<EOF > /etc/apt/sources.list.d/docker.sources
Types: deb
URIs: https://download.docker.com/linux/ubuntu
Suites: ${UBUNTU_CODENAME_DETECTED}
Components: stable
Architectures: $(dpkg --print-architecture)
Signed-By: /etc/apt/keyrings/docker.asc
EOF

apt_get update
apt_get install -y --no-install-recommends \
    docker-ce \
    docker-ce-cli \
    containerd.io \
    docker-buildx-plugin \
    docker-compose-plugin

# --- Docker daemon configuration ---
print_message "${YELLOW}" "Configuring the Docker daemon..."
mkdir -p /etc/docker

# OWNERSHIP POLICY: this script owns /etc/docker/daemon.json on RailsFast
# hosts (it provisioned Docker in the first place — v2 wrote this file too,
# so upgrades must be able to replace it). Hand-tuned changes are preserved
# in a timestamped .bak next to it and the replacement is diff-aware and
# validated before any restart. Everything else this script touches uses
# NON-destructive drop-ins (sshd_config.d, fail2ban jail.d, apt.conf.d).
#
# Deliberately NOT set here (so future maintainers don't "harden" these in):
#   - "userns-remap": CIS recommends it, but it shifts the uid mapping of
#     every bind mount — Kamal accessories' `directories:` (e.g. Postgres
#     data) would change ownership and break existing servers.
#   - "icc": false — only affects Docker's DEFAULT bridge; Kamal runs
#     everything on its own named network ("kamal"), so this hardens nothing
#     here and can surprise remote-builder/buildx setups.
#   - "ip": "127.0.0.1" — would make kamal-proxy's `--publish 80:80/443:443`
#     bind to loopback only and take the site off the internet.
#   - "hosts": never expose the Docker API on TCP; the local unix socket is
#     the only safe default (root-equivalent API).
DAEMON_JSON_CHANGED=false
DAEMON_JSON_TMP=$(mktemp)
# What each setting is for:
#   log-driver/log-opts: json-file is Docker's default driver; unrotated it
#     grows unbounded (max-size defaults to -1, max-file to 1). 100m x 3 keeps
#     ~300MB per container so `kamal app logs` has useful history without
#     eating a small disk. Rotation MUST live here, not in logrotate: "These
#     files are designed to be exclusively accessed by the Docker daemon" —
#     https://docs.docker.com/engine/logging/drivers/json-file/ and
#     https://docs.docker.com/engine/logging/configure/
#   live-restore: containers keep running while dockerd restarts (e.g. package
#     upgrades) — https://docs.docker.com/engine/daemon/live-restore/
#     CAVEAT: only guaranteed across patch releases of the engine.
#   userland-proxy false: publish ports purely via iptables DNAT instead of
#     one proxy process per published port (kept from v2, battle-tested on
#     RailsFast hosts). CAVEAT: curling a published port via 127.0.0.1 ON the
#     host itself can behave differently than via the public IP.
#   no-new-privileges: containers can't gain privileges via setuid binaries
#     (docker run reference, security section). Images that STEP DOWN from
#     root (postgres, kartoza/pg-backup use gosu/su-exec) still work — this
#     blocks privilege GAIN, not drop.
#   default-ulimits nofile 64000: Puma + Postgres under load exhaust the
#     1024 default fast; 64000 hard/soft is ample without being silly.
#   builder.gc keeps build cache bounded on remote-builder hosts (this same
#     script sets those up; deploy-only hosts barely use build cache).
cat <<'EOF' > "$DAEMON_JSON_TMP"
{
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "100m",
        "max-file": "3"
    },
    "live-restore": true,
    "userland-proxy": false,
    "no-new-privileges": true,
    "default-ulimits": {
        "nofile": {
            "Name": "nofile",
            "Hard": 64000,
            "Soft": 64000
        }
    },
    "builder": {
        "gc": {
            "enabled": true,
            "defaultKeepStorage": "20GB"
        }
    }
}
EOF

# Diff-aware: unchanged config on re-runs means no daemon restart at all.
if [[ ! -f /etc/docker/daemon.json ]] || ! cmp -s "$DAEMON_JSON_TMP" /etc/docker/daemon.json; then
    if [[ -f /etc/docker/daemon.json ]]; then
        cp /etc/docker/daemon.json "/etc/docker/daemon.json.bak-$(date +%Y%m%d%H%M%S)"
        print_message "${YELLOW}" "Existing daemon.json backed up next to it."
    fi
    mv "$DAEMON_JSON_TMP" /etc/docker/daemon.json
    chmod 644 /etc/docker/daemon.json

    # Validate BEFORE restarting so we never leave Docker unable to boot.
    # `dockerd --validate` parses the config and exits (added in Engine 23.0:
    # https://docs.docker.com/reference/cli/dockerd/ and
    # https://docs.docker.com/engine/release-notes/23.0/).
    if ! dockerd --validate --config-file=/etc/docker/daemon.json >/dev/null 2>&1; then
        print_error "The new /etc/docker/daemon.json failed validation. Restoring previous config."
        latest_backup=$(ls -t /etc/docker/daemon.json.bak-* 2>/dev/null | head -1 || true)
        if [[ -n "${latest_backup}" ]]; then
            cp "$latest_backup" /etc/docker/daemon.json
        else
            rm -f /etc/docker/daemon.json
        fi
        exit 1
    fi
    DAEMON_JSON_CHANGED=true
else
    rm -f "$DAEMON_JSON_TMP"
fi

# --now: also STARTS the daemon if it's stopped — with an unchanged config
# the restart branch below is skipped, and a plain `enable` would leave a
# manually-stopped daemon down.
systemctl enable --now docker
if [[ "$DAEMON_JSON_CHANGED" == "true" ]]; then
    # Safe on upgrade re-runs too: live-restore was already on in v2's config,
    # so running containers survive this restart.
    systemctl restart docker || {
        print_error "Docker failed to start. Logs:"
        journalctl -u docker.service --no-pager | tail -n 50
        exit 1
    }
fi

if ! docker info >/dev/null 2>&1; then
    print_error "Docker is not responding. Logs:"
    journalctl -u docker.service --no-pager | tail -n 50
    exit 1
fi

# --- Deploy user ---
# SECURITY REALITY CHECK: membership in the docker group is root-equivalent
# ("only trusted users should be allowed to control your Docker daemon" —
# https://docs.docker.com/engine/security/#docker-daemon-attack-surface).
# The deploy user is NOT a lesser-privileged account; it's root with a
# different spelling. Its value is operational (key-only, no password, what
# Kamal expects) — do not hand its key to anything you wouldn't hand root.
print_message "${YELLOW}" "Creating the '${DOCKER_USERNAME}' deploy user..."
if ! id -u "$DOCKER_USERNAME" >/dev/null 2>&1; then
    # --system --group reuses the docker group the docker-ce package already
    # created ("If that identically named group is not already present, it is
    # created" — i.e. an existing one is used as-is), and exits 0 if the user
    # already exists with compatible attributes, so re-runs are safe:
    # https://manpages.ubuntu.com/manpages/noble/en/man8/adduser.8.html
    adduser --system --group --shell /bin/bash --home "/home/${DOCKER_USERNAME}" --disabled-password "$DOCKER_USERNAME"
fi
usermod -aG docker "$DOCKER_USERNAME"

mkdir -p "/home/${DOCKER_USERNAME}/.ssh"
# 750, not 755: no other unprivileged users should exist on this host, but
# there's no reason for world-traversable homes either. sshd's StrictModes
# only requires the home not be group/world-WRITABLE.
chmod 750 "/home/${DOCKER_USERNAME}"

# Seed the deploy user's SSH keys (first run only — we don't clobber keys you
# may have added since). Preference order:
#   1. SSH_PUBLIC_KEY env var — explicit provisioning, best for unattended
#      custom setups.
#   2. $SUDO_USER's keys — on images whose provisioning user is not root
#      (e.g. 'ubuntu' on EC2), root's authorized_keys often contains only a
#      command="echo Please login as ubuntu" line that would break Kamal.
#   3. root's keys — the normal case on Hetzner/DO-style images where the
#      panel key lands in /root/.ssh/authorized_keys before cloud-init runcmd.
#
# When copying (2/3) we take only UNRESTRICTED key lines — lines that START
# with a key type. In authorized_keys grammar, options (command="...",
# no-port-forwarding, restrict, ...) come before the key type, so this grep
# skips them by construction: forced-command keys would hijack every Kamal
# command, and no-port-forwarding would break the local-registry tunnel.
# Grammar: sshd(8) AUTHORIZED_KEYS FILE FORMAT — https://man.openbsd.org/sshd#AUTHORIZED_KEYS_FILE_FORMAT
DEPLOY_KEYS_FILE="/home/${DOCKER_USERNAME}/.ssh/authorized_keys"

# "Usable" = the file contains at least one line that BOTH starts directly
# with a key type (i.e. carries no options — see the copy-filter rationale
# above) AND parses as a real key per ssh-keygen. Checking the file with
# ssh-keygen alone is NOT enough: it happily fingerprints forced-command and
# no-port-forwarding keys (verified empirically), and a v2-era server copied
# root's file wholesale — so a restricted-only file must not count as usable,
# or we'd disable password auth on a box Kamal still can't operate.
has_usable_key() {
    [[ -s "$1" ]] || return 1
    local tmp
    tmp=$(mktemp)
    if ! grep -E '^(ssh-(ed25519|rsa)|ecdsa-sha2|sk-ssh|sk-ecdsa)' "$1" > "$tmp" 2>/dev/null || [[ ! -s "$tmp" ]]; then
        rm -f "$tmp"
        return 1
    fi
    if ssh-keygen -lf "$tmp" >/dev/null 2>&1; then
        rm -f "$tmp"
        return 0
    fi
    rm -f "$tmp"
    return 1
}

# An explicit SSH_PUBLIC_KEY is ensured present even when a file already
# exists (append, deduped) — it's the operator's escape from a file full of
# unusable v2-copied restricted keys.
if [[ -n "$SSH_PUBLIC_KEY" ]] && ! grep -qxF "$SSH_PUBLIC_KEY" "$DEPLOY_KEYS_FILE" 2>/dev/null; then
    printf '%s\n' "$SSH_PUBLIC_KEY" >> "$DEPLOY_KEYS_FILE"
fi

if [[ ! -f "$DEPLOY_KEYS_FILE" ]]; then
    KEY_SOURCES=()
    [[ -n "${SUDO_USER:-}" ]] && KEY_SOURCES+=("/home/${SUDO_USER}/.ssh/authorized_keys")
    KEY_SOURCES+=("/root/.ssh/authorized_keys")
    for src in "${KEY_SOURCES[@]}"; do
        [[ -s "$src" ]] || continue
        if grep -E '^(ssh-(ed25519|rsa)|ecdsa-sha2|sk-ssh|sk-ecdsa)' "$src" > "${DEPLOY_KEYS_FILE}.tmp" 2>/dev/null && [[ -s "${DEPLOY_KEYS_FILE}.tmp" ]]; then
            mv "${DEPLOY_KEYS_FILE}.tmp" "$DEPLOY_KEYS_FILE"
            break
        fi
        rm -f "${DEPLOY_KEYS_FILE}.tmp"
    done
fi
chown -R "${DOCKER_USERNAME}:${DOCKER_USERNAME}" "/home/${DOCKER_USERNAME}/.ssh"
chmod 700 "/home/${DOCKER_USERNAME}/.ssh"
[[ -f "$DEPLOY_KEYS_FILE" ]] && chmod 600 "$DEPLOY_KEYS_FILE"

# --- SSH hardening ---
print_message "${YELLOW}" "Hardening SSH..."

# Only disable password login when a key is actually installed — otherwise a
# password-provisioned server would lock everyone out: the deploy user has no
# password (--disabled-password), so root's password login is the only door
# left until a key exists. PermitRootLogin must stay "yes" in that state too,
# because "prohibit-password" refuses exactly the login they'd need.
PASSWORD_AUTH="no"
PERMIT_ROOT="prohibit-password"
SSH_DEFERRED="false"
if ! has_usable_key "$DEPLOY_KEYS_FILE"; then
    # Configure the SAFE state (password login stays possible) so nobody gets
    # locked out — but this is an INCOMPLETE setup: unless ALLOW_INCOMPLETE_SSH
    # =true, final verification will fail it (exit 1, no success line, no
    # cloud-init reboot). A key-less host can't be deployed to by Kamal anyway.
    PASSWORD_AUTH="yes"
    PERMIT_ROOT="yes"
    SSH_DEFERRED="true"
    print_warning "No usable SSH key found for ${DOCKER_USERNAME} (or root). Password login stays ENABLED."
    print_warning "Install a key and re-run this script to finish:"
    print_warning "  from your machine:  ssh-copy-id root@this-server"
    print_warning "  then here:          bash railsfast-setup.sh"
    print_warning "  (or re-run with SSH_PUBLIC_KEY=\"ssh-ed25519 AAAA...\" to provision one explicitly)"
fi

# Upgrade path: v2 overwrote /etc/ssh/sshd_config and pinned 2016-era crypto
# lists there. Those pins replace OpenSSH's defaults entirely (any list
# without a +/-/^ prefix does: https://man.openbsd.org/sshd_config), which
# silently disabled the post-quantum key exchange that IS the default now —
# sntrup761x25519 on 24.04/OpenSSH 9.6, mlkem768x25519-sha256 on 26.04/
# OpenSSH 10.2 (https://manpages.ubuntu.com/manpages/noble/en/man5/sshd_config.5.html
# and .../resolute/...). Ubuntu's maintained defaults are strictly stronger
# than the pins, so remove them (exact-match only — user customizations with
# different values are left alone).
V2_PIN_LINES=(
    'KexAlgorithms curve25519-sha256@libssh.org,ecdh-sha2-nistp521,ecdh-sha2-nistp384,ecdh-sha2-nistp256'
    'Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr'
    'MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com'
)
for pin in "${V2_PIN_LINES[@]}"; do
    if grep -qxF "$pin" /etc/ssh/sshd_config; then
        grep -vxF "$pin" /etc/ssh/sshd_config > /etc/ssh/sshd_config.tmp
        cat /etc/ssh/sshd_config.tmp > /etc/ssh/sshd_config
        rm -f /etc/ssh/sshd_config.tmp
        print_message "${YELLOW}" "Removed stale v2 crypto pin from sshd_config: ${pin%% *}"
    fi
done

# Keep the user we came in through (if the script was run with sudo from a
# provisioning user like 'ubuntu', blocking them in AllowUsers = lockout).
ALLOWED_USERS="${DOCKER_USERNAME} root"
if [[ -n "${SUDO_USER:-}" && "${SUDO_USER}" != "root" && "${SUDO_USER}" != "${DOCKER_USERNAME}" ]]; then
    ALLOWED_USERS="${ALLOWED_USERS} ${SUDO_USER}"
fi

# WHY A DROP-IN, AND WHY 00-: sshd uses the FIRST value it reads for each
# keyword ("for each keyword, the first obtained value will be used" —
# https://man.openbsd.org/sshd_config), and Ubuntu's stock sshd_config has
# `Include /etc/ssh/sshd_config.d/*.conf` at the TOP, glob expanded in
# lexical order. So 00-railsfast.conf outranks cloud-init's
# 50-cloud-init.conf AND everything in the main file — no matter what they
# say. This is the fix for v2's real vulnerability: cloud-init writes
# `PasswordAuthentication yes` into 50-cloud-init.conf on password-
# provisioned servers (cc_set_passwords + ssh_util:
# https://github.com/canonical/cloud-init/blob/main/cloudinit/config/cc_set_passwords.py),
# and because v2's rewritten sshd_config ALSO started with that Include, the
# drop-in silently re-enabled password auth over v2's "no". Reproduced and
# verified on 24.04 before fixing (effective config said yes; v2's file said no).
#
# No `Port` directive on purpose: the default is 22 and Port lines ACCUMULATE
# (multiple Port lines = listen on multiple ports) rather than override — a
# v2-upgraded server already has `Port 22` in the main file.
mkdir -p /etc/ssh/sshd_config.d
# Transactional: keep the previous drop-in so a failed validation restores
# the last-known-good hardening instead of leaving none at all.
DROPIN=/etc/ssh/sshd_config.d/00-railsfast.conf
[[ -f "$DROPIN" ]] && cp "$DROPIN" "${DROPIN}.prev"
cat <<EOF > /etc/ssh/sshd_config.d/00-railsfast.conf
# RailsFast/Kamal SSH hardening (setup script v${SCRIPT_VERSION})
# This file intentionally sorts first: sshd keeps the first value it reads.
# Managed by the RailsFast server setup script — edits here survive re-runs
# only if you also disable the SSH section; prefer 01-*.conf for your own
# overrides of anything NOT set in this file.

# v2 pinned "AddressFamily inet" (IPv4 only), which makes sshd unreachable on
# IPv6-only VPSes. "any" is the OpenSSH default.
AddressFamily any

# VERBOSE logs the fingerprint of every key that authenticates — essential
# forensics on a multi-key deploy host; INFO only says "Accepted publickey".
LogLevel VERBOSE

# Key-only root, kept as break-glass (this is v2's prod-proven stance; set to
# "no" once you're confident you'll never need root directly — Kamal only
# needs the deploy user). When no key exists yet, the script sets "yes"
# temporarily so password-provisioned servers aren't locked out (see above).
PermitRootLogin ${PERMIT_ROOT}
AllowUsers ${ALLOWED_USERS}

PubkeyAuthentication yes
PasswordAuthentication ${PASSWORD_AUTH}
# Modern name for ChallengeResponseAuthentication (deprecated alias since
# OpenSSH 8.7: https://www.openssh.com/txt/release-8.7)
KbdInteractiveAuthentication no
PermitEmptyPasswords no
HostbasedAuthentication no
IgnoreRhosts yes
# Required on Ubuntu for account/session handling even with password auth off
UsePAM yes
StrictModes yes

LoginGraceTime 30
# 10, not the CIS-style 3-4: with password auth off, extra tries only cost
# log lines, and every key your ssh-agent offers counts as one "try" — devs
# with several keys in their agent would hit "Too many authentication
# failures" at lower values. fail2ban (below) handles the actual abuse.
MaxAuthTries 10
MaxSessions 10
ClientAliveInterval 300
ClientAliveCountMax 2
TCPKeepAlive no

# LOAD-BEARING for Kamal — do NOT "harden" this to no/local: Kamal's local
# registry mode (registry.server: localhost:PORT in deploy.yml, the RailsFast
# default) pushes images through an SSH REVERSE tunnel. Kamal calls net-ssh's
# forward.remote(), i.e. ssh -R, so the server's sshd must permit remote TCP
# forwarding. Verified in Kamal's source (not documented on kamal-deploy.org):
# https://github.com/basecamp/kamal/blob/main/lib/kamal/cli/build/port_forwarding.rb
AllowTcpForwarding yes
# The default, pinned: with GatewayPorts enabled, that same reverse-tunnel
# listener could bind the server's PUBLIC interface instead of 127.0.0.1 —
# briefly exposing your dev machine's registry to the internet mid-deploy.
# "no" forces all remote-forward listeners onto loopback.
# https://man.openbsd.org/sshd_config#GatewayPorts
GatewayPorts no
AllowAgentForwarding no
X11Forwarding no
PermitTTY yes
PrintMotd no
EOF

# Validate before applying — never break the door we came in through.
# `sshd -t`: "check the validity of the configuration file and sanity of the
# keys" (https://man.openbsd.org/sshd).
rollback_dropin() {
    if [[ -f "${DROPIN}.prev" ]]; then
        mv "${DROPIN}.prev" "$DROPIN"   # restore last-known-good hardening
    else
        rm -f "$DROPIN"                 # first run: back to distro config
    fi
    sshd -t || print_error "Rolled-back config also fails validation — inspect /etc/ssh manually before disconnecting!"
    systemctl try-reload-or-restart ssh || true
}

if ! sshd -t; then
    print_error "sshd rejected the new configuration. Rolling back."
    rollback_dropin
    exit 1
fi
# NOTE: ${DROPIN}.prev is kept until the reload AND the effective-config
# verification below both pass — the whole change is transactional.

# Ubuntu's sshd has been SOCKET-ACTIVATED since 22.10 (ssh.socket:
# https://discourse.ubuntu.com/t/sshd-now-uses-socket-based-activation-ubuntu-22-10-and-later/30189),
# so during cloud-init ssh.service is often NOT running yet. try-reload-or-
# restart reloads it when active and — unlike plain `reload` — cleanly does
# nothing when inactive (the next connection activates sshd fresh, reading
# the new config). Never `restart ... || true` (Tim's fork): that masks a
# broken config until the machine reboots into an unreachable sshd.
if ! systemctl try-reload-or-restart ssh; then
    print_error "sshd failed to reload with the new configuration. Rolling back."
    rollback_dropin
    exit 1
fi

# Verify the EFFECTIVE config — what sshd actually resolved from all files —
# not what any one file says. This is the check v2 got wrong (it grepped the
# file it had just written, and passed while cloud-init overrode it).
# `sshd -T`: "Extended test mode. Check the validity of the configuration
# file, output the effective configuration to stdout" (man sshd).
verify_sshd_effective() {
    local key=$1 expected_regex=$2
    local actual
    actual=$(sshd -T 2>/dev/null | awk -v k="$key" '$1==k{print $2; exit}')
    if ! echo "$actual" | grep -Eq "^(${expected_regex})$"; then
        print_error "sshd effective ${key} is '${actual}' (expected ${expected_regex})"
        return 1
    fi
    return 0
}

# Older OpenSSH prints prohibit-password as its legacy spelling
# "without-password" in -T output — accept both.
PERMIT_ROOT_EXPECTED="prohibit-password|without-password"
[[ "$PERMIT_ROOT" == "yes" ]] && PERMIT_ROOT_EXPECTED="yes"

SSHD_OK=true
verify_sshd_effective "passwordauthentication" "$PASSWORD_AUTH" || SSHD_OK=false
verify_sshd_effective "permitrootlogin" "$PERMIT_ROOT_EXPECTED" || SSHD_OK=false
verify_sshd_effective "kbdinteractiveauthentication" "no" || SSHD_OK=false
verify_sshd_effective "allowtcpforwarding" "yes" || SSHD_OK=false
verify_sshd_effective "gatewayports" "no" || SSHD_OK=false
verify_sshd_effective "x11forwarding" "no" || SSHD_OK=false
if [[ "$SSHD_OK" != "true" ]]; then
    # FATAL, not a warning: an effective-config mismatch means some other file
    # is overriding us — believing hardening applied when it didn't is exactly
    # the v2 vulnerability this script exists to fix. Roll back to the last-
    # known-good drop-in so we don't leave a half-applied state behind.
    print_error "Effective sshd configuration does not match what was just written. Rolling back."
    print_error "Most likely another file in /etc/ssh/sshd_config.d/ sorts BEFORE 00-railsfast.conf"
    print_error "(sshd keeps the FIRST value it reads; lower-sorting filenames win). Inspect with: sshd -T"
    rollback_dropin
    exit 1
fi
rm -f "${DROPIN}.prev"   # transaction committed
print_success "SSH effective configuration verified (sshd -T)"

# The EFFECTIVE port list (usually just "22", but a pre-customized server may
# use others; Port can appear multiple times = listen on all of them). The
# firewall and fail2ban sections below use this — opening only 22/tcp on a
# custom-port server would lock its owner out at the firewall.
SSH_EFFECTIVE_PORTS=$(sshd -T 2>/dev/null | awk '$1=="port"{print $2}' | sort -un)
[[ -z "$SSH_EFFECTIVE_PORTS" ]] && SSH_EFFECTIVE_PORTS="22"
SSH_PORTS_CSV=$(echo "$SSH_EFFECTIVE_PORTS" | paste -sd, -)
SSH_FIRST_PORT=$(echo "$SSH_EFFECTIVE_PORTS" | head -1)

# --- Firewall ---
print_message "${YELLOW}" "Configuring firewall..."
# No `ufw reset` (v2 did `ufw --force reset`): a reset wipes rules users
# added themselves (mosh, wireguard, monitoring). We only assert our baseline;
# every rule below is idempotent on re-runs.
ufw default deny incoming
ufw default allow outgoing
# `allow`, deliberately NOT `ufw limit`: limit throttles to 6 connections/30s
# per source, and Kamal opens several SSH connections in quick bursts during
# deploys (accessory boot, app boot, proxy) — rate-limiting the deploy IP
# makes deploys flaky. Brute force is already handled by key-only auth +
# fail2ban. And we open the EFFECTIVE sshd port(s), not a hardcoded 22: on a
# server that was pre-configured with a custom Port, allowing only 22 would
# lock the owner out the moment ufw turns on.
for ssh_port in $SSH_EFFECTIVE_PORTS; do
    ufw allow "${ssh_port}/tcp"
done
ufw allow 80/tcp
ufw allow 443/tcp
if [[ "$INSTALL_MOSH" == "true" ]]; then
    ufw allow 60000:61000/udp
fi
# --force just skips the "may disrupt existing connections" y/N prompt
# (cloud-init has no TTY); it's safe mid-SSH because 22/tcp is allowed above.
ufw --force enable

# HONESTY SECTION — ufw is NOT the whole firewall story on a Docker host:
# "traffic to and from that container gets diverted before it goes through
# the ufw firewall settings" (https://docs.docker.com/engine/network/packet-filtering-firewalls/).
# Ports PUBLISHED by containers bypass ufw entirely; host-level filtering for
# them belongs in the DOCKER-USER iptables chain
# (https://docs.docker.com/engine/network/firewall-iptables/).
# In the default Kamal + RailsFast topology this is fine: only kamal-proxy
# publishes ports (80/443, intended public); accessories without a `port:`
# key (the RailsFast postgres/pg-backup defaults) publish NOTHING and are
# reachable only on the private "kamal" Docker network
# (https://kamal-deploy.org/docs/configuration/proxy/ and .../accessories/).
print_warning "Heads up: ports PUBLISHED by Docker containers bypass ufw (Docker writes its own iptables rules)."
print_warning "A default Kamal + RailsFast setup only publishes 80/443 (kamal-proxy), so you're fine — but"
print_warning "set up your cloud provider's firewall (e.g. Hetzner Cloud Firewall: allow 22, 80, 443) as the outer layer,"
print_warning "and bind any custom accessory ports to 127.0.0.1 (e.g. \"127.0.0.1:9200:9200\") in deploy.yml."

# --- fail2ban ---
print_message "${YELLOW}" "Configuring fail2ban..."

# v2 shipped a speculative "docker" jail whose failregex ("failed login
# attempt from <HOST>") matches nothing any real service logs — remove the
# filter (exact v2 content only) and v2's jail.local that referenced it
# (a jail pointing at a missing filter stops fail2ban from starting).
if [[ -f /etc/fail2ban/filter.d/docker.conf ]] && grep -q 'failed login attempt from <HOST>' /etc/fail2ban/filter.d/docker.conf; then
    rm -f /etc/fail2ban/filter.d/docker.conf
fi
# Surgical: strip ONLY the [docker] section from jail.local (wherever it
# sits), keeping any jails an admin added to that file. A whole-file move
# would throw their config away with v2's.
if [[ -f /etc/fail2ban/jail.local ]] \
   && grep -q '^\[docker\]' /etc/fail2ban/jail.local \
   && grep -q 'filter = docker' /etc/fail2ban/jail.local; then
    cp /etc/fail2ban/jail.local "/etc/fail2ban/jail.local.bak-$(date +%Y%m%d%H%M%S)"
    awk 'BEGIN{skip=0} /^\[/{skip=($0=="[docker]")?1:0} !skip' /etc/fail2ban/jail.local > /etc/fail2ban/jail.local.tmp
    mv /etc/fail2ban/jail.local.tmp /etc/fail2ban/jail.local
    print_message "${YELLOW}" "Removed v2's [docker] jail from jail.local (it matched nothing and referenced a removed filter; a jail pointing at a missing filter stops fail2ban). Backup saved alongside."
fi

# Our config lives in jail.d — NOT jail.local, which belongs to the admin.
# fail2ban reads: jail.conf -> jail.d/*.conf (alphabetical) -> jail.local ->
# jail.d/*.local, and the LAST value wins (the opposite of sshd!) — so this
# file overrides the distro's jail.d/defaults-debian.conf ("d" < "r"), while
# any jail.local YOU write still overrides us. See man 5 jail.conf:
# https://manpages.ubuntu.com/manpages/noble/man5/jail.conf.5.html
cat <<EOF > /etc/fail2ban/jail.d/railsfast.conf
# RailsFast server setup (v${SCRIPT_VERSION}) — override in /etc/fail2ban/jail.local
[DEFAULT]
# Incremental banning: repeat offenders get exponentially longer bans, capped
# at a week. Supported since fail2ban 0.11 (24.04 ships 1.0.2, 26.04 ships
# 1.1.0). First offense: 1h.
bantime = 3600
bantime.increment = true
bantime.maxtime = 1w
findtime = 600
# 10 tries in 10 min before a ban: forgiving enough that an owner fumbling
# keys doesn't ban themselves off their own box (recoverable only via the
# provider console), tight enough for scanner noise. Key-only auth means
# these attempts are cryptographically hopeless anyway.
maxretry = 10
banaction = ufw
banaction_allports = ufw

[sshd]
enabled = true
# Journal backend scoped to THIS jail (not [DEFAULT]: a global backend=systemd
# would silently break any file-based jails an admin adds — the systemd
# backend ignores logpath). Reads auth events from the systemd journal, which
# works identically on 24.04/26.04 whether or not rsyslog//var/log/auth.log
# exists (24.04 minimal images ship without rsyslog). Note Ubuntu 24.04's own
# defaults-debian.conf sets backend=systemd globally anyway; scoping ours is
# still the politer pattern and matches what 25.10+ ships.
backend = systemd
# Watch the port(s) sshd actually listens on (default jails assume "ssh"=22)
port = ${SSH_PORTS_CSV}
# GOTCHA (the reason this line exists): Ubuntu's SSH unit is ssh.service, but
# fail2ban's stock sshd filter journal-matches _SYSTEMD_UNIT=sshd.service —
# on 24.04 (fail2ban 1.0.2) the jail can silently match nothing and never ban
# (fail2ban#1341: https://github.com/fail2ban/fail2ban/issues/1341, Debian
# #862348). This is the exact match Ubuntu 25.10+/Debian trixie ship as the
# fix in jail.d/defaults-debian.conf — a harmless duplicate there, the actual
# fix on 24.04:
# https://git.launchpad.net/ubuntu/+source/fail2ban/plain/debian/debian-files/jail.d_defaults-debian.conf?h=ubuntu/noble
journalmatch = _SYSTEMD_UNIT=ssh.service + _COMM=sshd
EOF

systemctl enable fail2ban
systemctl restart fail2ban

# --- auditd (optional) ---
# Off by default: an audit trail nobody ships anywhere or reads is disk churn
# and false confidence. Turn it on if you actually collect/review audit logs.
if [[ "$INSTALL_AUDITD" == "true" ]]; then
    print_message "${YELLOW}" "Installing auditd (INSTALL_AUDITD=true)..."
    apt_get install -y --no-install-recommends auditd audispd-plugins
    # v2 watched all of /var/lib/docker — every image pull/layer write became
    # audit events. Watch the things that change rarely and matter instead.
    cat <<'EOF' > /etc/audit/rules.d/docker.rules
# Watch Docker configuration and binaries (not /var/lib/docker: too noisy)
-w /usr/bin/dockerd -p x -k docker
-w /etc/docker -p wa -k docker
-w /etc/docker/daemon.json -p wa -k docker
-w /usr/lib/systemd/system/docker.service -p wa -k docker
-w /usr/bin/docker -p x -k docker-bin
EOF
    systemctl enable --now auditd
    augenrules --load || auditctl -R /etc/audit/rules.d/docker.rules
fi

# --- Automatic security updates ---
print_message "${YELLOW}" "Configuring automatic security updates..."
# Ubuntu's stock /etc/apt/apt.conf.d/50unattended-upgrades already enables
# the -security origins ("Right after installation, automatic installation
# of security updates will be enabled" —
# https://ubuntu.com/server/docs/how-to/software/automatic-updates/).
# v2 OVERWROTE that distro file; we add preferences in a drop-in instead.
# apt reads apt.conf.d in alphanumeric order and later files override scalar
# values, so 52-* wins over 50-* for exactly the two keys we set:
# https://manpages.ubuntu.com/manpages/noble/en/man5/apt.conf.5.html
cat <<'EOF' > /etc/apt/apt.conf.d/52unattended-upgrades-railsfast
// RailsFast server setup: opinions on top of Ubuntu's defaults.
// Automatic-Reboot stays false: an unsupervised 4am reboot is a bigger
// availability risk than a manually scheduled one. TRADE-OFF: kernel
// security updates are NOT live until you reboot. Either reboot when the
// MOTD says so, or enable Ubuntu Pro's livepatch (free for up to 5 machines:
// https://ubuntu.com/pro) — or flip this to "true" plus
// Unattended-Upgrade::Automatic-Reboot-Time "04:00"; if brief planned
// downtime is fine (Kamal containers restart on boot automatically).
Unattended-Upgrade::Remove-Unused-Dependencies "true";
Unattended-Upgrade::Automatic-Reboot "false";
EOF

# Guarantee the daily run is on (cloud images ship 20auto-upgrades enabled;
# minimal images may not). Written as OUR OWN drop-in — 21 sorts after 20, so
# these scalars win without touching the debconf-managed 20auto-upgrades, and
# an admin's 99-* file would still win over us. "1" = daily, per APT::Periodic.
cat <<'EOF' > /etc/apt/apt.conf.d/21auto-upgrades-railsfast
APT::Periodic::Update-Package-Lists "1";
APT::Periodic::Unattended-Upgrade "1";
EOF

# NOTE: packages from third-party repos are NOT auto-upgraded — "Just adding
# another package repository ... WILL NOT make unattended-upgrades consider
# it for updates" (Ubuntu docs above). That includes Docker's repo, and we
# deliberately DON'T add its origin ("Docker:${distro_codename}" — the repo's
# Release file says Origin: Docker, Label: Docker CE): a surprise dockerd
# restart is worse than running `apt upgrade` yourself now and then, and
# live-restore keeps containers up either way. Add it to
# Unattended-Upgrade::Allowed-Origins yourself if you prefer auto-updates.

# --- journald: persistent + size-capped ---
# Container logs land in the systemd journal when the app uses the journald
# Docker log driver, which RailsFast's config/deploy.yml sets by default so app
# logs SURVIVE Kamal's per-deploy container pruning (a per-container json-file is
# deleted with its container; the journal is host-level and outlives it). Two
# settings matter:
#   - Storage=persistent guarantees /var/log/journal exists, so logs survive a
#     reboot. The stock default (Storage=auto) is persistent ONLY if that
#     directory already exists — not guaranteed on a fresh server.
#   - SystemMaxUse caps journal disk use. journald's own default is 10% of the
#     filesystem (up to 4G); 5G holds a generous window of app history (weeks
#     for most apps) while staying a small fraction of a typical 40GB+ prod
#     disk, leaving room for Postgres and images. journald hard-caps at this
#     value and vacuums older entries, so it can never exceed it. Lower it on
#     very small disks (keep it well under ~15% of the filesystem); raise it to
#     retain more.
mkdir -p /etc/systemd/journald.conf.d
cat <<'EOF' > /etc/systemd/journald.conf.d/99-railsfast.conf
[Journal]
Storage=persistent
SystemMaxUse=5G
EOF
systemctl restart systemd-journald

# --- Remove v2 footguns (upgrade mode) ---
remove_v2_leftovers() {
    local removed=false

    # THE big one. Kamal retains the last 5 containers + images per app for
    # `kamal rollback` (retain_containers default 5:
    # https://github.com/basecamp/kamal/blob/main/lib/kamal/configuration.rb)
    # and already prunes automatically after every successful deploy
    # (deploy invokes kamal:cli:prune:all:
    # https://github.com/basecamp/kamal/blob/main/lib/kamal/cli/main.rb,
    # https://kamal-deploy.org/docs/commands/prune/). v2's nightly
    # `docker system prune -af --volumes` deleted ALL stopped containers —
    # which is exactly what the retained rollback versions are — plus their
    # images and any anonymous volumes
    # (https://docs.docker.com/reference/cli/docker/system/prune/), so the
    # cron was both redundant AND broke `kamal rollback`.
    if [[ -f /etc/cron.daily/docker-cleanup ]]; then
        rm -f /etc/cron.daily/docker-cleanup
        print_message "${YELLOW}" "Removed v2's daily docker prune cron: it deleted the stopped containers/images Kamal keeps for 'kamal rollback'. Kamal prunes after each deploy on its own (keeps last 5)."
        removed=true
    fi
    # Docker's json log files are "designed to be exclusively accessed by the
    # Docker daemon" (https://docs.docker.com/engine/logging/configure/) —
    # v2's logrotate copytruncate rule raced the daemon on live files.
    # Rotation now lives in daemon.json only.
    if [[ -f /etc/logrotate.d/docker-logs ]]; then
        rm -f /etc/logrotate.d/docker-logs
        print_message "${YELLOW}" "Removed v2's logrotate rule for Docker json logs (Docker rotates them itself via daemon.json; copytruncate could corrupt them)."
        removed=true
    fi
    if [[ -f /etc/security/limits.d/docker.conf ]]; then
        rm -f /etc/security/limits.d/docker.conf
        print_message "${YELLOW}" "Removed v2's limits.d/docker.conf (container limits come from daemon.json default-ulimits)."
        removed=true
    fi

    # Don't auto-purge packages the user might conceivably have adopted —
    # print the recipe instead.
    if dpkg -l 2>/dev/null | grep -qE '^ii\s+(aide|rkhunter|logwatch)\s'; then
        print_warning "v2 installed aide/rkhunter/logwatch (plus postfix & friends via their dependencies). They do nothing useful without configured mail delivery."
        print_warning "If you never configured them, you can remove them with:"
        print_warning "  apt-get purge --autoremove -y aide aide-common rkhunter logwatch"
        print_warning "  apt-get purge --autoremove -y postfix   # only if you don't use local mail"
    fi

    [[ "$removed" == "true" ]] && print_success "v2 leftovers cleaned up"
    return 0
}
remove_v2_leftovers

# --- Clean up (BEFORE verification: a hiccup here must not be able to
# follow a success verdict). No `apt autoremove`: on repurposed boxes it can
# remove packages the admin still wants, and the ongoing case is handled by
# Unattended-Upgrade::Remove-Unused-Dependencies above. ---
apt_get clean

# --- Facts for the final report (computed BEFORE verification; all fail-soft)
# Primary outbound IPv4 (source address of the default route) — NOT
# `hostname -I | awk '{print $1}'`, whose first entry can be docker0's
# 172.17.0.1 depending on interface enumeration. Every step tolerates failure
# (|| true) or `set -e -o pipefail` would abort the whole run on an
# IPv6-only host, where `ip -4 route get` exits non-zero.
SERVER_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}') || true
if [[ -z "${SERVER_IP:-}" ]]; then
    SERVER_IP=$(ip -6 route get 2606:4700:4700::1111 2>/dev/null | awk '{for(i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}') || true
fi
if [[ -z "${SERVER_IP:-}" ]]; then
    SERVER_IP=$(hostname -I 2>/dev/null | awk '{print $1}') || true
fi
SERVER_IP="${SERVER_IP:-<this-server-ip>}"

SWAP_INFO=$(swapon --show --noheadings 2>/dev/null | awk '{print $1, $3}' | head -1) || true

# --- Final verification ---
print_message "${YELLOW}" "Verifying setup..."
verify_setup() {
    local failed=0

    # Kernel parameters (spot check)
    local params=(
        "kernel.unprivileged_bpf_disabled=1"
        "net.ipv4.ip_forward=1"
        "fs.protected_hardlinks=1"
        "fs.protected_symlinks=1"
    )
    for param in "${params[@]}"; do
        local name=${param%=*}
        local expected=${param#*=}
        local actual
        actual=$(sysctl -n "$name" 2>/dev/null || echo "NOT_FOUND")
        if [[ "$actual" != "$expected" ]]; then
            print_error "Kernel parameter $name = $actual (expected $expected)"
            failed=1
        fi
    done

    # Services
    local services=("docker" "fail2ban" "ufw")
    [[ "$INSTALL_AUDITD" == "true" ]] && services+=("auditd")
    for service in "${services[@]}"; do
        if ! systemctl is-active --quiet "$service"; then
            print_error "Service $service is not running"
            failed=1
        fi
    done

    # Docker works, and works for the deploy user — the one thing Kamal
    # actually needs. runuser (util-linux, always present) instead of sudo:
    # it doesn't depend on sudoers policy.
    if ! docker info 2>/dev/null | grep -q "Live Restore Enabled: true"; then
        print_error "Docker live-restore is not enabled"
        failed=1
    fi
    if ! runuser -u "$DOCKER_USERNAME" -- docker version >/dev/null 2>&1; then
        print_error "User '${DOCKER_USERNAME}' cannot talk to the Docker daemon"
        failed=1
    fi

    # SSH keys for the deploy user: FATAL by default — a key-less host can't
    # be deployed to by Kamal, and completing "successfully" without one is a
    # lie (second-review fix). ALLOW_INCOMPLETE_SSH=true is the explicit
    # escape hatch for deliberate password-only bootstraps; even then the
    # canonical success line is withheld at the end.
    if ! has_usable_key "/home/${DOCKER_USERNAME}/.ssh/authorized_keys"; then
        if [[ "$SSH_DEFERRED" == "true" && "$ALLOW_INCOMPLETE_SSH" == "true" ]]; then
            print_warning "No usable SSH key for '${DOCKER_USERNAME}' — accepted because ALLOW_INCOMPLETE_SSH=true"
        else
            print_error "No usable SSH key installed for '${DOCKER_USERNAME}' — Kamal cannot connect to this server."
            print_error "Install one (see the instructions printed above) and re-run. If you truly need a key-less"
            print_error "bootstrap, re-run with ALLOW_INCOMPLETE_SSH=true."
            failed=1
        fi
    fi

    # Firewall
    if ! ufw status | grep -q "Status: active"; then
        print_error "UFW firewall is not active"
        failed=1
    fi

    # fail2ban sshd jail actually loaded (config parse errors leave the
    # service up with the jail dead — check the jail, not the service)
    if ! fail2ban-client status sshd >/dev/null 2>&1; then
        print_error "fail2ban sshd jail is not loaded"
        failed=1
    fi

    # Time sync (warning only: NTP can legitimately take a minute after boot)
    if [[ "$(timedatectl show -p NTPSynchronized --value 2>/dev/null)" != "yes" ]]; then
        print_warning "Clock not synchronized yet (this can take a minute after boot)"
    fi

    return $failed
}

# FATAL on failure (a release-review fix): printing success over a broken
# setup was v2's disease. On failure we exit non-zero — under the RailsFast
# cloud-init flow (`... && reboot`) the server then does NOT reboot, the
# success line never appears, and the log shows exactly which checks failed.
if verify_setup; then
    print_success "All verification checks passed"
else
    print_error "Setup verification FAILED — this server is NOT ready."
    print_error "Fix the failed checks above and re-run this script (it is safe to re-run; completed steps are skipped or re-asserted)."
    exit 1
fi

# The version marker co-travels with the final verdict: nothing that can fail
# stands between here and the completion line below.
echo "${SCRIPT_VERSION}" > "$VERSION_MARKER"

# --- Done ---
# CONTRACT: the RailsFast quickstart tells users to wait for the literal text
# "SUCCESS: Setup complete!" in the cloud-init log — that line must print
# ONLY for a fully verified setup. The ALLOW_INCOMPLETE_SSH escape hatch
# passes verification, but deliberately does NOT get the canonical line
# (so cloud-init flows can't sail past a key-less server unnoticed).
if [[ "$SSH_DEFERRED" == "true" ]]; then
    print_warning "Setup finished WITHOUT SSH hardening (ALLOW_INCOMPLETE_SSH=true):"
    print_warning "password login is still enabled and Kamal has no deploy key yet."
else
    print_success "Setup complete! System hardening successful."
fi
echo ""
print_message "${GREEN}" "System information:"
echo "  Docker:  $(docker --version)"
echo "  Kernel:  $(uname -r)"
echo "  UFW:     $(ufw status | head -1)"
echo "  Swap:    ${SWAP_INFO:-none}"
echo "  IP:      ${SERVER_IP}"
echo ""
SSH_HINT="ssh ${DOCKER_USERNAME}@${SERVER_IP}"
[[ "$SSH_FIRST_PORT" != "22" ]] && SSH_HINT="ssh -p ${SSH_FIRST_PORT} ${DOCKER_USERNAME}@${SERVER_IP}"

print_message "${YELLOW}" "Next steps:"
print_message "${YELLOW}" "1. From your OWN machine, verify you can connect BEFORE closing this session:"
print_message "${YELLOW}" "     ${SSH_HINT}"
print_message "${YELLOW}" "2. Point your app at this server (RailsFast: config/deploy.yml -> servers, accessories, ssh.user: ${DOCKER_USERNAME})"
print_message "${YELLOW}" "3. Add your provider's firewall (e.g. Hetzner Cloud Firewall): allow ${SSH_PORTS_CSV}, 80, 443"
print_message "${YELLOW}" "4. Reboot when convenient to load the updated kernel (cloud-init setups reboot automatically)"
print_message "${YELLOW}" "5. Deploy: kamal accessory boot all && kamal registry setup && kamal deploy"
print_message "${YELLOW}" "6. Audit this server anytime (read-only checks): wget -qO verify.sh https://verify.railsfast.com && bash verify.sh"
if [[ "$PASSWORD_AUTH" == "yes" ]]; then
    print_warning "REMINDER: SSH hardening is NOT finished — password login is still enabled because no usable"
    print_warning "SSH key was found. Install a key (ssh-copy-id root@${SERVER_IP}) and re-run this script."
fi
echo ""
echo -e "${GREEN}────────────────────────────────────────────────────────────────────${NC}"
echo -e "${GREEN} This script is free & open source, from the maker of RailsFast.${NC}"
echo -e "${GREEN}${NC}"
echo -e "${GREEN} If you're not already using RailsFast: it's the Ruby on Rails${NC}"
echo -e "${GREEN} starter that takes you from zero to a production SaaS in minutes —${NC}"
echo -e "${GREEN} auth, payments, SSL, deploys... all wired up on a \$5 VPS like this.${NC}"
echo -e "${GREEN}${NC}"
echo -e "${GREEN} This script saved you an afternoon; RailsFast saves you weeks:${NC}"
echo -e "${GREEN}   --> https://railsfast.com${NC}"
echo -e "${GREEN}────────────────────────────────────────────────────────────────────${NC}"

exit 0