zscripts-token-savers/bash/zdeploy

#!/usr/bin/env bash
# Evomedia.net Token Savers — https://github.com/kellymichels/zscripts-token-savers
# Created by Kelly Michels · dev@evomedia.net
# Licensed under the MIT License. See LICENSE.
#
# zdeploy — deploy any project defined in zconfig.json to the server.
# Each project runs its own docker compose stack; the handler is picked by the
# project's "kind": python | vite | nextjs | edge | docker.
#
# Usage:
#   zdeploy <project> [<project> ...] [--note "message"]
#   zdeploy all                       # every project (edge kinds first), stop at first failure
#
# Flow (python/vite/nextjs): zip source -> free server disk space -> scp up ->
# unzip into remote.path (preserving server-side .env) -> docker compose build + up
# -> verify the live site reports the new build version. Zips are always deleted.

set -uo pipefail
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/zhelpers.sh"
z_track_start "$@"
z_need_config
z_require ssh scp zip curl jq

projects=(); NOTE="Build deployed"
while [ $# -gt 0 ]; do
  case "$1" in
    --note) NOTE="${2:-Build deployed}"; shift 2 ;;
    *)      projects+=("${1#-}"); shift ;;   # tolerate switch-style keys (zdeploy -myproject)
  esac
done

if [ "${#projects[@]}" -eq 0 ]; then
  printf '\n'; warn "Usage: zdeploy <project> [<project> ...] | all  [--note \"message\"]"
  dim "  Projects in zconfig.json: $(zproj_csv)"
  dim "  'all' deploys everything (edge kinds first) and stops at the first failure."
  exit 1
fi

EC2_IP="$(zec2_ip)"
STACK_ROOT="$(zq '.ec2.stackRoot')"
REMOTE_HOME="/home/$(zec2_user)"
EC2_USER="$(zec2_user)"
TEMP_ROOT="$(zq '.paths.temp')"
{ [ -z "$TEMP_ROOT" ] || [ "$TEMP_ROOT" = "null" ]; } && TEMP_ROOT="${TMPDIR:-/tmp}"
mkdir -p "$TEMP_ROOT"

# 'all' -> every project, edge kinds first (the proxy must route before apps ship).
if printf '%s\n' "${projects[@]}" | grep -qx 'all'; then
  edge_keys=(); while IFS= read -r _zl || [ -n "$_zl" ]; do edge_keys+=("$_zl"); done \
    < <(jq -r '.projects | to_entries[] | select((.key | startswith("_") | not) and .value.kind == "edge") | .key' "$ZCONFIG")
  rest_keys=(); while IFS= read -r _zl || [ -n "$_zl" ]; do rest_keys+=("$_zl"); done \
    < <(jq -r '.projects | to_entries[] | select((.key | startswith("_") | not) and .value.kind != "edge") | .key' "$ZCONFIG")
  projects=("${edge_keys[@]+"${edge_keys[@]}"}" "${rest_keys[@]+"${rest_keys[@]}"}")
  info "Deploying all projects: $(printf '%s ' "${projects[@]}")"
fi

deploy_zip_name() {  # <key>
  local zn; zn="$(zproj "$1" .deploy.zipName)"
  [ -n "$zn" ] && printf '%s' "$zn" || printf '%sDeploy.zip' "$1"
}

# Pre-upload cleanup: remove stale deploy zips, prune docker, truncate big logs,
# fail if under 1.5 GB free.
preflight_cleanup() {  # <extra_zip_to_remove...>
  printf '\n'; info "--- [Preflight] Freeing disk space on the server ---"
  local rm_clause="true"
  [ $# -gt 0 ] && rm_clause="rm -f $*"
  local cmd
  cmd="echo '--- df / before cleanup ---'; df -h /"
  cmd+="; echo '--- removing stale deploy artifacts ---'; $rm_clause"
  cmd+="; echo '--- pruning docker build cache + dangling images + stopped containers ---'"
  cmd+="; sudo docker container prune -f >/dev/null 2>&1 || true"
  cmd+="; sudo docker builder prune -f >/dev/null 2>&1 || true"
  cmd+="; sudo docker image prune -af >/dev/null 2>&1 || true"
  cmd+="; echo '--- truncating large container logs ---'"
  cmd+="; sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true"
  cmd+="; echo '--- df / after cleanup ---'; df -h /"
  cmd+='; avail_mb=$(df --output=avail -BM / | tail -n 1 | tr -dc 0-9)'
  cmd+='; [ -z "$avail_mb" ] && avail_mb=0'
  cmd+='; echo available_mb=$avail_mb'
  cmd+='; if [ "$avail_mb" -lt 1500 ]; then echo "ERROR: less than 1.5 GB free on /. Grow the root volume or run: sudo docker system prune -af" >&2; exit 11; fi'
  zssh "$cmd" || { err "Server pre-flight cleanup failed. Root volume too full (need ~1.5 GB free, ideally 3+)."; return 1; }
}

# Post-deploy cleanup: prune what this deploy created; running stacks untouched.
post_cleanup() {  # <label>
  printf '\n'; info "--- [Post-deploy] Reclaiming disk space ($1) ---"
  local cmd
  cmd="sudo docker container prune -f >/dev/null 2>&1 || true"
  cmd+="; sudo docker builder prune -f >/dev/null 2>&1 || true"
  cmd+="; sudo docker image prune -af >/dev/null 2>&1 || true"
  cmd+="; sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true"
  cmd+="; df -h /"
  zssh "$cmd" || warn "  Post-deploy cleanup returned non-zero; continuing."
}

send_zip() {  # <local_zip> <zip_name>
  scp -i "$(zec2_pem)" "$1" "$(zec2_target):$REMOTE_HOME/" \
    || { err "SCP upload failed. Likely server disk space. Try: rm -f $REMOTE_HOME/$2"; return 1; }
}

remote_unzip() {  # <zip_name> <dest_path>
  zec2_step "unzip $1" \
    "test -f $REMOTE_HOME/$1 || { echo 'missing $REMOTE_HOME/$1'; exit 2; }; unzip -t $REMOTE_HOME/$1 || exit 3; unzip -o $REMOTE_HOME/$1 -d $2; uc=\$?; if [ \$uc -gt 1 ]; then exit \$uc; fi; exit 0"
}

# ── Deploy verification (build-version match, not just HTTP 200 — a 200 can be
#    a stale cached build; the version match proves the new build is live) ─────

wait_verify_api() {  # <key> <expected_label> <timeout_sec>
  local key="$1" expected="$2" timeout="${3:-60}" domain live deadline
  domain="$(zproj "$key" .domain)"
  printf '\n'; info "--- [$key] Live build verification (expect $expected) ---"
  deadline=$((SECONDS + timeout))
  while [ $SECONDS -lt $deadline ]; do
    live="$(http_get "http://$EC2_IP/api/build-version" "$domain" | jq -r '.build_version // empty' 2>/dev/null)"
    if [ -n "$live" ]; then
      if [ "$live" = "$expected" ]; then
        ok "  PASS - live build $live matches expected."
        return 0
      fi
      warn "  Live build is $live, expected $expected - waiting..."
    else
      dim "  /api/build-version not ready yet - waiting..."
    fi
    sleep 3
  done
  warn "  WARNING: live build did not match $expected within ${timeout}s (a stale build may be cached)."
  return 1
}

wait_verify_static() {  # <key> <pre_product_version> <pre_build_number>
  local key="$1" pv="$2" bn="$3" expected container domain live deadline raw
  if [ -z "$pv" ] || [ -z "$bn" ]; then
    printf '\n'; warn "--- [$key version] SKIPPED (no local build-version.json - see 'Enabling deploy verification' in README) ---"
    return 0
  fi
  expected="v${pv}.$((bn + 1))"
  container="$(zproj "$key" .remote.containerName)"
  domain="$(zproj "$key" .domain)"
  printf '\n'; info "--- [$key] Live build verification (expect $expected) ---"
  deadline=$((SECONDS + 45))
  while [ $SECONDS -lt $deadline ]; do
    if [ -n "$container" ]; then
      # build-version.json may be blocked externally by the edge proxy;
      # read it inside the running container instead.
      raw="$(zssh "sudo docker exec $container cat /usr/share/nginx/html/build-version.json 2>/dev/null" 2>/dev/null)"
    else
      raw="$(http_get "http://$EC2_IP/build-version.json" "$domain")"
    fi
    live="$(printf '%s' "$raw" | jq -r '"v\(.productVersion).\(.buildNumber)"' 2>/dev/null)"
    if [ -n "$live" ] && [ "$live" != "null" ] && [ "$live" != "v." ]; then
      if [ "$live" = "$expected" ]; then
        ok "  PASS - live build $live matches expected."
        return 0
      fi
      warn "  Live build is $live, expected $expected - waiting..."
    else
      dim "  Container not ready yet - waiting..."
    fi
    sleep 3
  done
  warn "  WARNING: live build did not match $expected within 45s (upload or Docker build may have failed, or a stale build is cached)."
}

# Fast-forward the checkout before a deploy when deploy.gitPull is set — zdeploy
# zips the working tree, so a stale checkout would ship stale code while still
# bumping the build number. Abort rather than deploy uncertain code.
deploy_git_pull() {  # <key>
  local key="$1" root branch
  [ "$(zproj "$key" .deploy.gitPull)" = "true" ] || return 0
  root="$(zproj "$key" .localRoot)"
  if [ ! -d "$root/.git" ]; then
    warn "  gitPull set but '$root' is not a git repo - skipping pull."
    return 0
  fi
  printf '\n'; info "--- [0] git pull --ff-only ---"
  branch="$(cd "$root" && git rev-parse --abbrev-ref HEAD)"
  dim "  Branch: $branch"
  (cd "$root" && git pull --ff-only) || {
    err "git pull --ff-only failed in '$root' (branch '$branch'). Resolve it, then re-run - refusing to deploy possibly-stale code."
    return 1
  }
  dim "  Now at: $(cd "$root" && git log -1 --oneline)"
}

# ── Kind handlers ────────────────────────────────────────────────────────────

deploy_python() {  # <key>
  local key="$1" start=$SECONDS root remote_path compose_dir app_svc zip_name zip_local
  local bv="" has_version_tool=0 excl=() attempt out label domain
  root="$(zproj "$key" .localRoot)"
  remote_path="$(zproj "$key" .remote.path)"
  compose_dir="$(zremote_compose_dir "$key")"
  app_svc="$(zproj "$key" .remote.appService)"; [ -n "$app_svc" ] || app_svc="app"
  zip_name="$(deploy_zip_name "$key")"
  zip_local="$TEMP_ROOT/$zip_name"
  label="$(zproj "$key" .label)"
  domain="$(zproj "$key" .domain)"
  [ -f "$root/scripts/build_version_tool.py" ] && has_version_tool=1
  [ -d "$root" ] || { err "Project root not found: $root"; return 1; }

  printf '\n'; info "=== $label deploy (python) ==="
  dim "Local zip: $zip_local"

  printf '\n'; info "--- [1] Zipping $label ---"
  find "$root" -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null
  excl=(); while IFS= read -r _zl || [ -n "$_zl" ]; do excl+=("$_zl"); done < <(z_archive_excludes "$key" 0)
  z_archive "$root" "$zip_local" 0 "-" "${excl[@]}" || return 1

  preflight_cleanup "$REMOTE_HOME/$zip_name" || { rm -f "$zip_local"; return 1; }

  printf '\n'; info "--- [2] Uploading zip ---"
  send_zip "$zip_local" "$zip_name" || { rm -f "$zip_local"; return 1; }
  rm -f "$zip_local"

  printf '\n'; info "--- [3] Unzipping and rebuilding on the server ---"
  zec2_step "apt-get install unzip" "sudo apt-get update -qq && sudo apt-get install -y unzip" || return 1
  zec2_step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${EC2_USER}:${EC2_USER} $STACK_ROOT" || return 1
  zec2_step "ensure shared web network" "sudo docker network create web 2>/dev/null || true" || return 1
  zec2_step "backup .env if present" "if [ -f $remote_path/.env ]; then cp $remote_path/.env $REMOTE_HOME/.env.${key}_bak; fi" || return 1
  zec2_step "replace project directory" "sudo rm -rf $remote_path && sudo mkdir -p $remote_path && sudo chown ${EC2_USER}:${EC2_USER} $remote_path" || return 1
  remote_unzip "$zip_name" "$remote_path" || return 1
  zec2_step "restore .env from backup" "if [ -f $REMOTE_HOME/.env.${key}_bak ]; then cp $REMOTE_HOME/.env.${key}_bak $remote_path/.env; fi" || return 1
  zec2_step "require compose directory" "test -d $compose_dir" || return 1
  zec2_step "docker compose build $app_svc" "cd $compose_dir && sudo COMPOSE_BAKE=false docker compose build $app_svc" || return 1
  zec2_step "docker compose up -d" "cd $compose_dir && sudo COMPOSE_BAKE=false docker compose up -d" || return 1
  zec2_step "record deploy time; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remote_path/.last_deploy_utc > /dev/null && rm -f $REMOTE_HOME/$zip_name" || return 1

  if [ "$has_version_tool" -eq 1 ]; then
    printf '\n'; info "--- [4] Incrementing build version ---"
    for attempt in 1 2 3 4 5; do
      out="$(zssh "cd $compose_dir && sudo docker compose exec -T $app_svc python scripts/build_version_tool.py bump" 2>/dev/null | tail -1)"
      [ -n "$out" ] && { bv="$out"; break; }
      warn "  Attempt $attempt failed, retrying in 3s..."
      sleep 3
    done
    [ -n "$bv" ] || { err "Build version bump failed after 5 attempts"; return 1; }

    (cd "$root" && { python3 scripts/build_version_tool.py set "$bv" >/dev/null 2>&1 || python scripts/build_version_tool.py set "$bv" >/dev/null 2>&1; })
    zssh "echo '$bv' | sudo tee $remote_path/.build_version > /dev/null"
    if [ -f "$root/scripts/build_changelog_tool.py" ]; then
      (cd "$root" && { python3 scripts/build_changelog_tool.py append --version "$bv" --note "$NOTE" >/dev/null 2>&1 \
                       || python scripts/build_changelog_tool.py append --version "$bv" --note "$NOTE" >/dev/null 2>&1; })
    fi

    printf '\n'; info "--- [5] Restarting app to pick up new version ---"
    zssh "cd $compose_dir && sudo COMPOSE_BAKE=false docker compose restart $app_svc" \
      || { err "App restart after build bump failed"; return 1; }

    wait_verify_api "$key" "$bv" 30 || true
  else
    printf '\n'; info "--- [4] Basic reachability check (no build_version_tool - see 'Enabling deploy verification' in README) ---"
    local deadline=$((SECONDS + 30)) code up=0
    while [ $SECONDS -lt $deadline ]; do
      sleep 3
      code="$(http_code "http://$EC2_IP/" "$domain")"
      if [ "$code" != "000" ] && [ "$code" -lt 500 ]; then up=1; break; fi
      dim "  App not ready yet - waiting..."
    done
    if [ "$up" -eq 1 ]; then ok "  App is responding."; else warn "  WARNING: app did not respond within 30s."; fi
  fi

  post_cleanup "$key"

  local elapsed=$((SECONDS - start))
  printf '\n'; ok "--- [Done] $label deployed! ---"
  [ -n "$domain" ] && warn "Site: https://$domain"
  [ -n "$bv" ] && note "Build Version: $bv"
  info "Change Note: $NOTE"
  dim "Deploy Time: $(printf '%02d:%02d' $((elapsed / 60)) $((elapsed % 60))) (${elapsed}s)"
}

deploy_vite() {  # <key>
  local key="$1" start=$SECONDS root remote_path zip_name zip_local excl=()
  local pre_pv="" pre_bn="" label domain edge_pc
  root="$(zproj "$key" .localRoot)"
  remote_path="$(zproj "$key" .remote.path)"
  zip_name="$(deploy_zip_name "$key")"
  zip_local="$TEMP_ROOT/$zip_name"
  label="$(zproj "$key" .label)"
  domain="$(zproj "$key" .domain)"
  [ -d "$root" ] || { err "Project root not found: $root"; return 1; }

  printf '\n'; info "=== $label deploy (vite/static) ==="
  dim "Local zip: $zip_local"

  printf '\n'; info "--- [1] Zipping site ---"
  if [ -f "$root/build-version.json" ]; then
    pre_pv="$(jq -r '.productVersion // empty' "$root/build-version.json")"
    pre_bn="$(jq -r '.buildNumber // empty' "$root/build-version.json")"
    [ -n "$pre_pv" ] && dim "  Pre-zip build label: v${pre_pv}.${pre_bn} (server-side build will bump +1)"
  fi
  excl=(); while IFS= read -r _zl || [ -n "$_zl" ]; do excl+=("$_zl"); done < <(z_archive_excludes "$key" 0)
  z_archive "$root" "$zip_local" 0 "-" "${excl[@]}" || return 1

  preflight_cleanup "$REMOTE_HOME/$zip_name" || { rm -f "$zip_local"; return 1; }

  printf '\n'; info "--- [2] Uploading zip ---"
  send_zip "$zip_local" "$zip_name" || { rm -f "$zip_local"; return 1; }
  rm -f "$zip_local"

  printf '\n'; info "--- [3] Unzipping and rebuilding on the server ---"
  zec2_step "apt-get install unzip" "sudo apt-get update -qq && sudo apt-get install -y unzip" || return 1
  zec2_step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${EC2_USER}:${EC2_USER} $STACK_ROOT" || return 1
  zec2_step "ensure shared web network" "sudo docker network create web 2>/dev/null || true" || return 1
  zec2_step "replace project directory" "sudo rm -rf $remote_path && sudo mkdir -p $remote_path && sudo chown ${EC2_USER}:${EC2_USER} $remote_path" || return 1
  remote_unzip "$zip_name" "$remote_path" || return 1
  zec2_step "require compose file" "test -f $remote_path/docker-compose.yml" || return 1
  zec2_step "docker compose build" "cd $remote_path && sudo COMPOSE_BAKE=false docker compose build" || return 1
  zec2_step "docker compose up -d" "cd $remote_path && sudo COMPOSE_BAKE=false docker compose up -d" || return 1

  local ek; ek="$(zedge_key)"
  if [ -n "$ek" ]; then
    edge_pc="$(zproj "$ek" .proxyContainer)"
    [ -n "$edge_pc" ] && { zec2_step "reload edge nginx (flush DNS cache for new container IP)" "sudo docker exec $edge_pc nginx -s reload" || return 1; }
  fi
  zec2_step "record deploy time; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remote_path/.last_deploy_utc > /dev/null && rm -f $REMOTE_HOME/$zip_name" || return 1

  wait_verify_static "$key" "$pre_pv" "$pre_bn"
  post_cleanup "$key"

  local elapsed=$((SECONDS - start))
  printf '\n'; ok "--- [Done] $label deployed! ---"
  [ -n "$domain" ] && warn "Site: https://$domain"
  info "Change Note: $NOTE"
  dim "Deploy Time: $(printf '%02d:%02d' $((elapsed / 60)) $((elapsed % 60))) (${elapsed}s)"
}

deploy_next() {  # <key>
  local key="$1" start=$SECONDS root remote_path app_svc zip_name zip_local excl=()
  local pre_pv="" pre_bn="" label domain db_user db_name prod_port
  root="$(zproj "$key" .localRoot)"
  remote_path="$(zproj "$key" .remote.path)"
  app_svc="$(zproj "$key" .remote.appService)"; [ -n "$app_svc" ] || app_svc="web"
  zip_name="$(deploy_zip_name "$key")"
  zip_local="$TEMP_ROOT/$zip_name"
  label="$(zproj "$key" .label)"
  domain="$(zproj "$key" .domain)"
  [ -d "$root" ] || { err "Project root not found: $root"; return 1; }

  printf '\n'; info "=== $label deploy (nextjs) ==="
  dim "Local zip: $zip_local"

  if [ -f "$root/public/build-version.json" ]; then
    pre_pv="$(jq -r '.productVersion // empty' "$root/public/build-version.json")"
    pre_bn="$(jq -r '.buildNumber // empty' "$root/public/build-version.json")"
    [ -n "$pre_pv" ] && dim "  Pre-zip build label: v${pre_pv}.${pre_bn} (server-side build will bump +1)"
  fi

  printf '\n'; info "--- [1] Zipping project files ---"
  excl=(); while IFS= read -r _zl || [ -n "$_zl" ]; do excl+=("$_zl"); done < <(z_archive_excludes "$key" 0)
  z_archive "$root" "$zip_local" 0 "-" "${excl[@]}" || return 1

  preflight_cleanup "$REMOTE_HOME/$zip_name" || { rm -f "$zip_local"; return 1; }

  printf '\n'; info "--- [2] Uploading zip ---"
  send_zip "$zip_local" "$zip_name" || { rm -f "$zip_local"; return 1; }
  rm -f "$zip_local"

  printf '\n'; info "--- [3] Unzipping and rebuilding on the server ---"
  zec2_step "ensure unzip installed" "sudo apt-get update -qq && sudo apt-get install -y unzip" || return 1
  zec2_step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${EC2_USER}:${EC2_USER} $STACK_ROOT" || return 1
  zec2_step "ensure shared web network" "sudo docker network create web 2>/dev/null || true" || return 1
  zec2_step "backup .env if present" "if [ -f $remote_path/.env ]; then cp $remote_path/.env $REMOTE_HOME/.env.${key}_bak; fi" || return 1
  zec2_step "replace project directory" "sudo rm -rf $remote_path && sudo mkdir -p $remote_path && sudo chown ${EC2_USER}:${EC2_USER} $remote_path" || return 1
  remote_unzip "$zip_name" "$remote_path" || return 1
  zec2_step "restore .env from backup" "if [ -f $REMOTE_HOME/.env.${key}_bak ]; then cp $REMOTE_HOME/.env.${key}_bak $remote_path/.env; fi" || return 1

  printf '\n'; info "--- [4] Docker compose rebuild ---"
  zec2_step "docker compose down" "cd $remote_path && sudo COMPOSE_BAKE=false docker compose down" || return 1
  zec2_step "docker compose build" "cd $remote_path && sudo COMPOSE_BAKE=false docker compose build" || return 1
  zec2_step "docker compose up -d" "cd $remote_path && sudo COMPOSE_BAKE=false docker compose up -d" || return 1

  db_user="$(zproj "$key" .db.user)"; db_name="$(zproj "$key" .db.name)"
  if [ -n "$db_user" ] && [ -n "$db_name" ]; then
    zec2_step "wait for postgres ready" \
      "cd $remote_path && for i in \$(seq 1 30); do sudo docker compose exec -T db pg_isready -U $db_user -d $db_name >/dev/null 2>&1 && break; sleep 2; done" || return 1
  fi
  if [ "$(zproj "$key" .migrations)" = "prisma" ]; then
    zec2_step "apply prisma migrations" "cd $remote_path && sudo docker compose exec -T $app_svc npx prisma migrate deploy" || return 1
  fi
  zec2_step "record deploy timestamp; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remote_path/.last_deploy_utc > /dev/null && rm -f $REMOTE_HOME/$zip_name" || return 1

  printf '\n'; info "--- [5] Verifying deployment ---"
  prod_port="$(zproj "$key" .ports.prod)"
  if [[ "$prod_port" =~ ^[0-9]+$ ]]; then
    local direct="http://${EC2_IP}:${prod_port}/" deadline=$((SECONDS + 60)) code verified=0
    while [ $SECONDS -lt $deadline ]; do
      sleep 4
      code="$(http_code "$direct")"
      if [ "$code" = "200" ]; then
        ok "  PASS - app is responding at $direct"
        verified=1; break
      fi
      dim "  App not ready yet - waiting..."
    done
    [ "$verified" -eq 1 ] || warn "  WARNING: no response at $direct within 60s (is the port open in the security group?)."
  fi
  if [ -n "$pre_pv" ]; then
    wait_verify_api "$key" "v${pre_pv}.$((pre_bn + 1))" 60 || true
  else
    warn "  (No public/build-version.json - version verification skipped. See 'Enabling deploy verification' in README.)"
  fi

  post_cleanup "$key"

  local elapsed=$((SECONDS - start))
  printf '\n'; ok "--- [Done] $label deployed! ---"
  [ -n "$domain" ] && warn "Site:        https://$domain"
  info "Change Note: $NOTE"
  dim "Deploy Time: $(printf '%02d:%02d' $((elapsed / 60)) $((elapsed % 60))) (${elapsed}s)"
}

deploy_edge() {  # <key>
  local key="$1" root remote_path pc label cert_mount="" rm_stale="" f
  root="$(zproj "$key" .localRoot)"
  remote_path="$(zproj "$key" .remote.path)"
  pc="$(zproj "$key" .proxyContainer)"
  label="$(zproj "$key" .label)"

  printf '\n'; info "=== $label deploy (edge nginx ingress) ==="
  [ -d "$root" ] || { err "Edge root not found: $root"; return 1; }
  for f in docker-compose.yml nginx.conf; do
    [ -f "$root/$f" ] || { err "Missing $root/$f"; return 1; }
  done

  zec2_step "ensure shared web network" "sudo docker network create web 2>/dev/null || true" || return 1
  zec2_step "ensure edge dir" "sudo mkdir -p $remote_path && sudo chown ${EC2_USER}:${EC2_USER} $remote_path" || return 1

  # Ship every top-level file in the edge folder — nginx.conf, compose, css,
  # htpasswd, whatever the proxy serves. Subdirectories (logs, certs) stay put.
  while IFS= read -r -d '' f; do
    info "  >> uploading $(basename "$f")"
    scp -i "$(zec2_pem)" "$f" "$(zec2_target):$remote_path/" \
      || { err "SCP failed for $(basename "$f")"; return 1; }
  done < <(find "$root" -maxdepth 1 -type f ! -name nul -print0)

  local certs; certs="$(zproj "$key" .certsSource)"
  [ -n "$certs" ] && cert_mount="-v $certs:/etc/letsencrypt/:ro "
  zec2_step "validate new nginx.conf" \
    "sudo docker run --rm -v $remote_path/nginx.conf:/etc/nginx/nginx.conf:ro ${cert_mount}nginx:1.27-alpine nginx -t -c /etc/nginx/nginx.conf" || return 1

  # A container from an older compose project may still hold the proxy name;
  # docker refuses a second create with the same name, so remove it first.
  [ -n "$pc" ] && rm_stale="; sudo docker rm -f $pc 2>/dev/null || true"
  zec2_step "edge: compose down + remove stale proxy" "cd $remote_path && sudo docker compose down 2>/dev/null || true$rm_stale" || return 1
  zec2_step "edge compose up -d" "cd $remote_path && sudo docker compose up -d" || return 1
  [ -n "$pc" ] && { zec2_step "edge nginx reload" "sudo docker exec $pc nginx -s reload || true" || return 1; }
  zec2_step "fix nginx-logs permissions (if present)" \
    "if [ -d $remote_path/nginx-logs ]; then sudo chmod 777 $remote_path/nginx-logs; sudo chmod 666 $remote_path/nginx-logs/*.log 2>/dev/null || true; fi" || return 1
  ok "--- [Done] Edge proxy deploy finished ---"
}

deploy_docker() {  # <key>
  local key="$1" root remote_path label f domain
  root="$(zproj "$key" .localRoot)"
  remote_path="$(zproj "$key" .remote.path)"
  label="$(zproj "$key" .label)"
  domain="$(zproj "$key" .domain)"

  printf '\n'; info "=== $label deploy (docker compose) ==="
  [ -d "$root" ] || { err "Project root not found: $root"; return 1; }
  [ -f "$root/docker-compose.yml" ] || { err "Missing $root/docker-compose.yml"; return 1; }

  zec2_step "ensure shared web network" "sudo docker network create web 2>/dev/null || true" || return 1
  zec2_step "ensure project dir" "sudo mkdir -p $remote_path && sudo chown ${EC2_USER}:${EC2_USER} $remote_path" || return 1

  while IFS= read -r -d '' f; do
    info "  >> uploading $(basename "$f")"
    scp -i "$(zec2_pem)" "$f" "$(zec2_target):$remote_path/" \
      || { err "SCP failed for $(basename "$f")"; return 1; }
  done < <(find "$root" -maxdepth 1 -type f ! -name nul -print0)

  zec2_step "docker compose pull" "cd $remote_path && sudo docker compose pull" || return 1
  zec2_step "docker compose up -d" "cd $remote_path && sudo docker compose up -d" || return 1

  post_cleanup "$key"
  printf '\n'; ok "--- [Done] $label deploy finished ---"
  [ -n "$domain" ] && warn "Site: https://$domain"
}

# ── Dispatch ─────────────────────────────────────────────────────────────────

for key in "${projects[@]}"; do
  zproj_require "$key"
  deploy_git_pull "$key" || exit 1
  kind="$(zproj "$key" .kind)"
  case "$kind" in
    python) deploy_python "$key" || exit 1 ;;
    vite)   deploy_vite   "$key" || exit 1 ;;
    nextjs) deploy_next   "$key" || exit 1 ;;
    edge)   deploy_edge   "$key" || exit 1 ;;
    docker) deploy_docker "$key" || exit 1 ;;
    *)      err "No deploy handler for kind '$kind' (project '$key'). Add a deploy_<kind> function in zdeploy."; exit 1 ;;
  esac
done