zscripts-token-savers/bash/zec2online

#!/usr/bin/env bash
# Evomedia.net Token Savers — https://github.com/kellymichels/zscripts-token-savers
# Created by Kelly Michels · dev@evomedia.net
# Licensed under the MIT License. See LICENSE.
#
# zec2online — deep health check: verify apps are live AND running the expected
# build; auto-start downed stacks via docker compose and stream diagnostics.
#
# Usage:
#   zec2online                          # every project with a "domain" in zconfig.json
#   zec2online <project> [<project> ...]
#
# A plain HTTP-200 check passes even when a stale cached build is live — the
# local-vs-server version match is what proves the deployed build is the one running.

set -uo pipefail
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/zhelpers.sh"
z_track_start "$@"
z_need_config
z_require curl ssh

projects=(); host_override=""
while [ $# -gt 0 ]; do
  case "$1" in
    --host) host_override="${2:-}"; shift 2 ;;
    -*)     err "Unknown option: $1"; exit 1 ;;
    *)      projects+=("$1"); shift ;;
  esac
done

HOST="${host_override:-$(zec2_ip)}"
EDGE_KEY="$(zedge_key)"

if [ "${#projects[@]}" -eq 0 ]; then
  projects=(); while IFS= read -r _zl || [ -n "$_zl" ]; do projects+=("$_zl"); done < <(zproj_keys_with_domain)
  if [ "${#projects[@]}" -eq 0 ]; then
    warn "No projects with a 'domain' configured in zconfig.json."
    exit 1
  fi
fi

compare_versions() {  # <key> <local> <server>
  local key="$1" local_v="$2" server_v="$3"
  if [ "$local_v" = "unknown" ] && [ "$server_v" = "unknown" ]; then
    dim "  Version: unable to determine local or server version (see 'Enabling deploy verification' in README)"
  elif [ "$local_v" = "unknown" ]; then
    note "  Server:  $server_v"; dim "  Local:   unknown (could not read local build version)"
  elif [ "$server_v" = "unknown" ]; then
    note "  Local:   $local_v"; dim "  Server:  unknown (could not read server build version)"
  elif [ "$local_v" = "$server_v" ]; then
    ok "  Version: $server_v (local and server match)"
  else
    note "  Local:   $local_v"; note "  Server:  $server_v"
    warn "  WARNING: local and server versions differ - redeploy with: zdeploy $key"
  fi
}

test_http_online() {  # <host-header>
  local code
  code="$(http_code "http://${HOST}/" "$1")"
  if [ "$code" = "000" ]; then return 1; fi
  if [ "$code" -lt 500 ]; then
    if [ "$code" -ge 300 ]; then dim "  HTTP $code - redirect from app, server is live."
    else dim "  HTTP $code - service reachable."; fi
    return 0
  fi
  warn "  HTTP $code - unexpected server response."
  return 1
}

show_diagnostics() {  # <key> <reason>
  local key="$1" reason="$2" compose_dir cmd pc
  [ -f "$(zec2_pem)" ] || { dim "  Diagnostics skipped: PEM key not found at $(zec2_pem)"; return; }
  compose_dir="$(zremote_compose_dir "$key")"
  printf '\n'; note "  --- $key diagnostics ($reason) ---"
  cmd="echo '== services =='; cd $compose_dir 2>/dev/null && sudo docker compose ps"
  cmd+="; echo '== uptime / df =='; uptime; df -h /"
  cmd+="; echo '== oom =='; dmesg -T 2>/dev/null | grep -iE 'oom|killed' | tail -n 10"
  cmd+="; echo '== app logs (80) =='; cd $compose_dir 2>/dev/null && sudo docker compose logs --tail 80"
  if [ -n "$EDGE_KEY" ]; then
    pc="$(zproj "$EDGE_KEY" .proxyContainer)"
    if [ -n "$pc" ]; then
      cmd+="; echo '== edge proxy state =='; sudo docker ps --filter name=$pc --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'"
      cmd+="; echo '== edge proxy logs (40) =='; sudo docker logs --tail 40 $pc 2>&1 | tail -n 40"
    fi
  fi
  zssh "$cmd" || warn "  Diagnostics SSH failed."
  note "  --- end $key diagnostics ---"; printf '\n'
}

online_check() {  # <key>
  local key="$1" label domain compose_dir start_cmd edge_dir deadline
  label="$(zproj "$key" .label)"; [ -n "$label" ] || label="$key"
  domain="$(zproj "$key" .domain)"

  printf '\n'; info "=== zec2online: $label ($key) ==="
  dim "  Target: http://${HOST}/  (Host: $domain)"
  printf '\n'

  warn '  [1] Checking TCP + HTTP...'
  if test_http_online "$domain"; then
    compare_versions "$key" "$(local_version_label "$key")" "$(remote_version_label "$key" "$HOST")"
    ok "  UP - $label is running."
    printf '\n'
    return 0
  fi

  err '  DOWN - Service not responding.'
  show_diagnostics "$key" 'DOWN before recovery'

  warn '  [2] Starting stack (and edge proxy if defined)...'
  compose_dir="$(zremote_compose_dir "$key")"
  [ -f "$(zec2_pem)" ] || { err "  Cannot start: PEM key not found at: $(zec2_pem)"; return 1; }
  start_cmd="sudo docker network create web 2>/dev/null || true"
  start_cmd+="; if [ -f $compose_dir/docker-compose.yml ]; then cd $compose_dir && sudo docker compose up -d 2>&1 | tail -10; else echo 'compose missing at $compose_dir'; exit 2; fi"
  if [ -n "$EDGE_KEY" ]; then
    edge_dir="$(zproj "$EDGE_KEY" .remote.path)"
    [ -n "$edge_dir" ] && start_cmd+="; if [ -f $edge_dir/docker-compose.yml ]; then cd $edge_dir && sudo docker compose up -d 2>&1 | tail -10; fi"
  fi
  zssh "$start_cmd" || { err "  SSH failed. Check connectivity and PEM key."; return 1; }

  printf '\n'; warn '  [3] Waiting for service (up to 30s)...'
  deadline=$((SECONDS + 30))
  while [ $SECONDS -lt $deadline ]; do
    sleep 3
    dim '      checking...'
    if test_http_online "$domain"; then
      printf '\n'
      compare_versions "$key" "$(local_version_label "$key")" "$(remote_version_label "$key" "$HOST")"
      ok "  UP - $label is now running."
      return 0
    fi
  done

  printf '\n'; err '  TIMEOUT - Service did not respond within 30 seconds.'
  show_diagnostics "$key" 'recovery TIMEOUT'
  return 1
}

exit_code=0
for key in "${projects[@]}"; do
  zproj_require "$key"
  if [ -z "$(zproj "$key" .domain)" ]; then
    printf '\n'; warn "Skipping '$key' - no domain configured."
    continue
  fi
  online_check "$key" || exit_code=1
done
exit "$exit_code"