run_experiments.sh

#!/bin/bash
# run_experiments.sh
# Runs all four baselines serially. For each:
#   1. Start ./server <mode>
#   2. Start resource_monitor.py
#   3. Run load_tester.py (foreground, wait for completion)
#   4. Archive all logs into logs/exp_<mode>_<timestamp>/
#
# Usage:
#   ./run_experiments.sh            # run all 4 modes
#   ./run_experiments.sh ewma       # run a single mode
#
# CSCI 599: Network Systems for Cloud Computing
# University of Southern California

set -e
cd "$(dirname "$0")"

# ── Colour helpers ────────────────────────────────────────────────────────────
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
info()    { echo -e "${CYAN}[INFO]${NC} $*"; }
success() { echo -e "${GREEN}[OK]${NC}   $*"; }
warn()    { echo -e "${YELLOW}[WARN]${NC} $*"; }
error()   { echo -e "${RED}[ERR]${NC}  $*"; }

# ── Build ─────────────────────────────────────────────────────────────────────
info "Building server..."
make -s
success "Build OK"

# ── Experiment runner ─────────────────────────────────────────────────────────
# run_one <label> <server_args...>
#   label:       directory-safe name (e.g. "ewma", "static_15")
#   server_args: args passed to ./server (e.g. "static 15")
run_one() {
    local label="$1"; shift
    local server_args="$@"   # remaining args go to ./server

    echo ""
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    info "Experiment: ${label}  (./server ${server_args})"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    local exp_ts; exp_ts=$(date +%Y%m%d_%H%M%S)
    local exp_dir="logs/exp_${label}_${exp_ts}"
    mkdir -p "$exp_dir"

    # Kill ALL lingering gateway server processes by exact binary name.
    # pkill -x matches the process name field exactly (not cmdline substring),
    # so it only kills our compiled 'server' binary, not vscode or other tools.
    pkill -x server 2>/dev/null || true; sleep 1

    # Clean stale sockets from previous runs
    rm -f /tmp/faas_worker_*.sock /tmp/faas_template_ctrl.sock /tmp/faas_arima.sock
    # Remove previous server log for this mode (avoid confusion with archived copy)
    rm -f "logs/server_${label}.log"

    # ── Start server ──────────────────────────────────────────────────────────
    info "Starting server (mode=${label})..."
    ./server $server_args &
    local SERVER_PID=$!

    # Wait until template process signals READY (appears in server log).
    # Pillow import takes ~800ms; poll for up to 10s.
    local ready=0
    for i in $(seq 1 20); do
        sleep 0.5
        if ! kill -0 "$SERVER_PID" 2>/dev/null; then
            error "Server exited unexpectedly for mode '${label}'."
            return 1
        fi
        local log_file="logs/server_${label}.log"
        if [ -f "$log_file" ] && grep -q "READY\|starting\|Pre-warming" "$log_file" 2>/dev/null; then
            ready=1; break
        fi
    done
    if [ "$ready" -eq 0 ]; then
        warn "Server did not signal ready in 10s — proceeding anyway."
    fi
    # Extra buffer: let workers settle after READY
    sleep 1
    success "Server running (pid=${SERVER_PID})"

    # ── Start resource monitor ────────────────────────────────────────────────
    local resource_csv="${exp_dir}/resource.csv"
    info "Starting resource monitor → ${resource_csv}"
    python3 resource_monitor.py "$resource_csv" > "${exp_dir}/resource_monitor_output.txt" 2>&1 &
    local MONITOR_PID=$!
    sleep 0.5   # give monitor time to find the server process

    # ── Run load tester ───────────────────────────────────────────────────────
    info "Running load_tester.py..."
    local load_tester_log="${exp_dir}/load_tester_output.txt"
    # Use a sentinel file to reliably find the test log created during this run
    local sentinel; sentinel=$(mktemp logs/.sentinel.XXXXXX)
    python3 load_tester.py 2>&1 | tee "$load_tester_log"
    success "Load test complete"

    # ── Stop monitor and server ───────────────────────────────────────────────
    kill "$MONITOR_PID" 2>/dev/null; wait "$MONITOR_PID" 2>/dev/null || true
    kill "$SERVER_PID"  2>/dev/null; wait "$SERVER_PID"  2>/dev/null || true
    success "Server and monitor stopped"

    # ── Archive logs ──────────────────────────────────────────────────────────
    # Server log — mode label matches the log filename exactly
    # (e.g. label=ewma → server_ewma.log; label=static_15 → server_static_15.log)
    local server_log="logs/server_${label}.log"
    if [ -f "$server_log" ]; then
        mv "$server_log" "${exp_dir}/server.log"
    else
        warn "Server log not found: ${server_log}"
    fi

    # Find the test_*.log created after the sentinel file
    local test_log
    test_log=$(find logs -maxdepth 1 -name "test_*.log" -newer "$sentinel" 2>/dev/null \
               | sort | tail -1)
    rm -f "$sentinel"
    if [ -n "$test_log" ] && [ -f "$test_log" ]; then
        mv "$test_log" "${exp_dir}/test.log"
    else
        warn "Test log not found for mode '${label}'"
    fi

    success "Archived → ${exp_dir}/"
    echo "    server.log  test.log  resource.csv  load_tester_output.txt"
}

# ── Main ──────────────────────────────────────────────────────────────────────
OVERALL_TS=$(date +%Y%m%d_%H%M%S)

if [ $# -gt 0 ]; then
    # Single mode passed as argument
    mode="$1"
    case "$mode" in
        ewma)           run_one "ewma"           ewma ;;
        ewma_adaptive)  run_one "ewma_adaptive"  ewma_adaptive ;;
        reactive)       run_one "reactive"       reactive ;;
        static)         run_one "static_15"      static 15 ;;
        arima)          run_one "arima"          arima ;;
        *)  error "Unknown mode '$mode'. Valid: ewma ewma_adaptive reactive static arima"; exit 1 ;;
    esac
else
    # Run all five serially
    run_one "ewma"           ewma
    run_one "ewma_adaptive"  ewma_adaptive
    run_one "reactive"       reactive
    run_one "static_15"      static 15
    run_one "arima"          arima
fi

# ── Final summary ─────────────────────────────────────────────────────────────
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
success "All experiments complete. Results:"
ls -d logs/exp_*/  2>/dev/null | sort | while read d; do
    echo "    ${d}"
done
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"