mirror of
https://github.com/kellymichels/zscripts-token-savers
synced 2026-10-07 07:18:18 +00:00
Kelly Michels
f1b1fd6264
Build stamp catch-up for 1 merged PR(s) since v1.0.0.0.22:
135c585 docs(changelog): give every shipped release its own section, and generate the twin (#68)
1042 lines
58 KiB
PowerShell
1042 lines
58 KiB
PowerShell
# Evomedia.net Token Savers — https://github.com/evomedia-net/evo.zscripts
|
|
# Created by Kelly Michels · dev@evomedia.net
|
|
# Licensed under the MIT License. See LICENSE.
|
|
# Version: v1.0.0.0.23
|
|
|
|
# zdeploy.ps1 — deploy any project defined in zconfig.json to the server.
|
|
# Each project runs its own docker compose stack; the handler is picked by the
|
|
# project's "kind": python | vite | nextjs | edge | docker | static.
|
|
#
|
|
# Usage:
|
|
# zdeploy <project> [<project> ...] [-Note "message"]
|
|
# zdeploy all # ztokens first, then every project (edge kinds next), stop at first failure
|
|
# zdeploy ztokens # refresh the live-usage stats (see below)
|
|
#
|
|
# Examples:
|
|
# zdeploy viteapp
|
|
# zdeploy pyapp -Note "fix billing banner"
|
|
# zdeploy all -Note "weekly release"
|
|
# zdeploy ztokens evo # refresh token-stats.json, then ship the site with it
|
|
#
|
|
# "ztokens" is a pseudo-project, not a zconfig entry: it runs `ztokens -Publish`
|
|
# from the sibling ztokens repo, refreshing the token-stats.json the public
|
|
# zscripts page charts. `all` runs it first automatically; called standalone,
|
|
# list it before a site project (as above) so that project's deploy zip picks
|
|
# up the freshly written file.
|
|
#
|
|
# Flow (python/vite/nextjs): zip source -> free server disk space -> scp up ->
|
|
# unzip into remote.path (preserving server-side .env* files and anything in
|
|
# deploy.preserve) -> docker compose build + up -> verify the live site reports
|
|
# the new build version. Zips are always deleted.
|
|
#
|
|
# Compose service-name conventions (override with remote.appService):
|
|
# python kind: app service "app", db service "db"
|
|
# nextjs kind: app service "web", db service "db"
|
|
#
|
|
# Verification (python kind, when there is no scripts/build_version_tool.py):
|
|
# Projects with a "verify" block are checked ON the server via
|
|
# localhost:<port><path> — the only accurate way for stacks that are not
|
|
# published through the edge proxy. Projects with only a "domain" fall back
|
|
# to a Host-header request. Projects with neither are reported as NOT
|
|
# verified rather than passing on the proxy's default vhost.
|
|
#
|
|
param(
|
|
[Parameter(Position = 0, ValueFromRemainingArguments = $true)]
|
|
[string[]]$Projects = @(),
|
|
[string]$Note = "Build deployed"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
. (Join-Path $PSScriptRoot "ZHelpers.ps1")
|
|
Start-ZTracking
|
|
|
|
$cfg = Get-ZConfig
|
|
$EC2_IP = $cfg.ec2.ip
|
|
$PEM_KEY = $cfg.ec2.pemKey
|
|
$STACK_ROOT = $cfg.ec2.stackRoot
|
|
$SSH_TARGET = Get-Ec2Target
|
|
$SSH_OPTS = Get-Ec2SshOpts # see ZHelpers.ps1 - these are what stop a deploy hanging
|
|
$SCP_OPTS = Get-Ec2ScpOpts # same, minus -n: scp rejects it with a usage error
|
|
$RemoteHome = Get-Ec2Home
|
|
$Ec2User = $cfg.ec2.user
|
|
|
|
$TempRoot = $cfg.paths.temp
|
|
if (-not (Test-Path -LiteralPath $TempRoot)) {
|
|
New-Item -ItemType Directory -Path $TempRoot -Force | Out-Null
|
|
}
|
|
|
|
if ($Projects.Count -eq 0) {
|
|
$keys = (Get-ZProjectKeys) -join ', '
|
|
Write-Host ""
|
|
Write-Host "Usage: zdeploy <project> [<project> ...] | all | ztokens [-Note `"message`"]" -ForegroundColor Yellow
|
|
Write-Host " Projects in zconfig.json: $keys" -ForegroundColor Gray
|
|
Write-Host " 'all' deploys everything (edge kinds first) and stops at the first failure." -ForegroundColor Gray
|
|
Write-Host " 'ztokens' refreshes the live-usage stats published to the zscripts page." -ForegroundColor Gray
|
|
Stop-ZTracking; exit 1
|
|
}
|
|
|
|
# Tolerate switch-style args (zdeploy -myproject) from muscle memory.
|
|
$Projects = @($Projects | ForEach-Object { $_.TrimStart('-') })
|
|
|
|
if ($Projects -contains 'all') {
|
|
# 'ztokens' first so any site project deployed below picks up fresh stats.
|
|
$Projects = @('ztokens') + @(Get-ZProjectKeys)
|
|
}
|
|
|
|
# Edge kinds first, however the list was produced. This is a correctness
|
|
# property, not a convenience of 'all': the proxy has to route before the apps
|
|
# behind it ship, or there is a window where a new app is live behind stale
|
|
# routing. `zdeploy evo edge` reads as "these two, edge included" and used to
|
|
# do the risky order, because this sort only ran for 'all'.
|
|
# Order within each group is preserved, so an intentional sequence still holds
|
|
# — notably `zdeploy ztokens evo`, where ztokens must still precede evo.
|
|
$requested = @($Projects)
|
|
$edgeKeys = @($Projects | Where-Object { $cfg.projects.$_.kind -eq 'edge' })
|
|
$restKeys = @($Projects | Where-Object { $cfg.projects.$_.kind -ne 'edge' })
|
|
$Projects = @($edgeKeys + $restKeys)
|
|
|
|
if ($Projects.Count -gt 1) {
|
|
# Say so when the order changed, so the reordering is never silent.
|
|
$note = if (($requested -join ',') -ne ($Projects -join ',')) { " (edge first)" } else { "" }
|
|
Write-Host "Deploying: $($Projects -join ', ')$note" -ForegroundColor Cyan
|
|
}
|
|
|
|
function Get-DeployZipName {
|
|
param([string]$Key, $Proj)
|
|
if ($Proj.deploy -and $Proj.deploy.zipName) { return $Proj.deploy.zipName }
|
|
return "${Key}Deploy.zip"
|
|
}
|
|
|
|
# Pre-upload cleanup: remove stale deploy zips, prune docker, truncate big logs,
|
|
# fail if under 1.5 GB free.
|
|
function Invoke-Ec2PreflightCleanup {
|
|
param([string[]]$ExtraZipsToRemove = @())
|
|
Write-Host "`n--- [Preflight] Freeing disk space on the server ---" -ForegroundColor Cyan
|
|
$rmZipsClause = if ($ExtraZipsToRemove.Count -gt 0) { "rm -f $($ExtraZipsToRemove -join ' ')" } else { "true" }
|
|
$preflightCmd = @(
|
|
"echo '--- df / before cleanup ---'",
|
|
"df -h /",
|
|
"echo '--- removing stale deploy artifacts ---'",
|
|
$rmZipsClause,
|
|
"echo '--- pruning docker build cache + dangling images + stopped containers ---'",
|
|
"sudo docker container prune -f >/dev/null 2>&1 || true",
|
|
"sudo docker builder prune -f >/dev/null 2>&1 || true",
|
|
"sudo docker image prune -af >/dev/null 2>&1 || true",
|
|
"echo '--- truncating large container logs ---'",
|
|
"sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true",
|
|
"echo '--- df / after cleanup ---'",
|
|
"df -h /",
|
|
"avail_mb=`$(df --output=avail -BM / | tail -n 1 | tr -dc 0-9)",
|
|
"[ -z `"`$avail_mb`" ] && avail_mb=0",
|
|
"echo available_mb=`$avail_mb",
|
|
"if [ `"`$avail_mb`" -lt 1500 ]; then echo 'ERROR: less than 1.5 GB free on /. Grow the root volume or run: sudo docker system prune -af' >&2; exit 11; fi"
|
|
) -join '; '
|
|
# Also through the wrapper (#119): the out-of-space branch above writes its
|
|
# ERROR to stderr and exits 11, so a bare ssh would surface a
|
|
# NativeCommandError instead of the actionable message below - exactly when
|
|
# the operator most needs to be told what to do. -FailHint keeps it.
|
|
Invoke-Ec2Step "server pre-flight cleanup" $preflightCmd `
|
|
-FailHint "Root volume too full (need ~1.5 GB free, ideally 3+). Grow it or run: sudo docker system prune -af"
|
|
}
|
|
|
|
# Post-deploy cleanup: prune build cache and dangling images created during this deploy.
|
|
# Containers/volumes still in use by the running stack are NOT touched.
|
|
function Invoke-Ec2PostDeployCleanup {
|
|
param([string]$Label = "post-deploy")
|
|
Write-Host "`n--- [Post-deploy] Reclaiming disk space ($Label) ---" -ForegroundColor Cyan
|
|
$cmd = @(
|
|
"sudo docker container prune -f >/dev/null 2>&1 || true",
|
|
"sudo docker builder prune -f >/dev/null 2>&1 || true",
|
|
"sudo docker image prune -af >/dev/null 2>&1 || true",
|
|
"sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true",
|
|
"df -h /"
|
|
) -join '; '
|
|
ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $cmd
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-Host " Post-deploy cleanup returned non-zero exit ($LASTEXITCODE); continuing." -ForegroundColor DarkYellow
|
|
}
|
|
}
|
|
|
|
function Send-DeployZip {
|
|
param([string]$LocalZip, [string]$ZipName)
|
|
scp @SCP_OPTS -i $PEM_KEY $LocalZip "${SSH_TARGET}:$RemoteHome/"
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "SCP upload failed (exit $LASTEXITCODE). Read scp's own output above: a usage block means bad arguments, not disk. If it is genuinely full, clear the stale archive: rm -f $RemoteHome/$ZipName"
|
|
}
|
|
}
|
|
|
|
function Invoke-RemoteUnzip {
|
|
param([string]$ZipName, [string]$DestPath)
|
|
$bash = 'test -f {2}/{0} || {{ echo "missing {2}/{0}"; exit 2; }}; unzip -t {2}/{0} || exit 3; unzip -o {2}/{0} -d {1}; uc=$?; if [ $uc -gt 1 ]; then exit $uc; fi; exit 0' -f $ZipName, $DestPath, $RemoteHome
|
|
Invoke-Ec2Step "unzip $ZipName" $bash
|
|
}
|
|
|
|
# ── Operator-file preservation (issue #2, hardened in #107) ──────────────────
|
|
# Deploys replace the project directory wholesale, which used to destroy every
|
|
# operator-managed file except ./.env. These helpers preserve all .env* files
|
|
# at the project root PLUS any paths listed in deploy.preserve (files or
|
|
# directories), by tarring them to the home dir before the wipe and extracting
|
|
# them back after the unzip. Server-side copies win over anything shipped in
|
|
# the zip — the same semantics ./.env always had.
|
|
#
|
|
# WHY THE ARCHIVE IS TIMESTAMPED AND NEVER DELETED (#107)
|
|
# -------------------------------------------------------
|
|
# This used to write one fixed preserve_<key>.tgz, and preserve's FIRST action
|
|
# was `rm -f` on it. That is the opposite of safe. The window between
|
|
# `sudo rm -rf` on the project directory and the restore step is the only time
|
|
# the tarball is the sole copy of the production secrets - and an interrupted
|
|
# run (dropped ssh, exit 255) stops exactly there, leaving a perfect backup
|
|
# behind. The next run then deleted that backup before doing anything else,
|
|
# tarred a directory that no longer had the files, and reported success.
|
|
# `2>/dev/null; true` on the tar is what made it silent.
|
|
#
|
|
# That destroyed civilcode's production deploy/.env on 2026-08-30. The site
|
|
# survived only because the running container still held its environment; a
|
|
# restart would have made the loss permanent.
|
|
#
|
|
# So, three independent changes, any one of which would have prevented it:
|
|
#
|
|
# 1. Each run writes its own preserve_<key>_<stamp>.tgz and restore no
|
|
# longer deletes it. Nothing removes an archive that has not been
|
|
# superseded - retention below prunes old ones instead.
|
|
# 2. Preserve first extracts any earlier archives with `tar -k`, which fills
|
|
# in files a previous interrupted run lost WITHOUT overwriting anything
|
|
# currently on disk. A hand-repaired .env therefore wins over the stale
|
|
# copy in the archive.
|
|
# 3. Preserve refuses to continue if it captured nothing while an earlier
|
|
# archive for the same key did have contents. Capturing zero files is
|
|
# normal for a project with no operator files (edge, gitea, landing) and
|
|
# catastrophic for one that has them; the prior archive is what tells the
|
|
# difference.
|
|
#
|
|
# One stamp per zdeploy process, so preserve and restore agree on the filename
|
|
# without threading it through every call site.
|
|
$script:PreserveStamp = Get-Date -Format 'yyyyMMdd-HHmmss'
|
|
$script:PreserveKeep = 5
|
|
|
|
function Get-PreserveTarball {
|
|
param([string]$Key)
|
|
"$RemoteHome/preserve_${Key}_$($script:PreserveStamp).tgz"
|
|
}
|
|
|
|
function Save-OperatorFiles {
|
|
param([string]$Key, $Proj, [string]$RemotePath)
|
|
$paths = @('.env*')
|
|
if ($Proj.deploy -and $Proj.deploy.preserve) { $paths += @($Proj.deploy.preserve) }
|
|
$spec = $paths -join ' '
|
|
$tarball = Get-PreserveTarball -Key $Key
|
|
$list = $tarball -replace '\.tgz$', '.list'
|
|
$glob = "$RemoteHome/preserve_${Key}_*"
|
|
$drop = $script:PreserveKeep + 1
|
|
|
|
# NOTE: no embedded double quotes or $( ) here - PowerShell 5.1 strips
|
|
# embedded double quotes when passing args to ssh.exe, silently corrupting
|
|
# the remote command, and $( ) would be evaluated locally. Remote shell
|
|
# variables are backtick-escaped so PowerShell leaves them alone. Globs
|
|
# expand remotely; tar's nonzero exit for missing paths is swallowed, but
|
|
# an empty capture is NOT (see the guard below).
|
|
$bash =
|
|
"cd $RemotePath || exit 9; " +
|
|
"for t in ${glob}.tgz; do [ -e `$t ] && tar -xzkf `$t -C $RemotePath 2>/dev/null; done; true; " +
|
|
"tar -czf $tarball $spec 2>/dev/null; " +
|
|
"tar -tzf $tarball > $list 2>/dev/null; " +
|
|
"if [ ! -s $list ]; then " +
|
|
"for p in ${glob}.list; do " +
|
|
"if [ -s `$p ] && [ `$p != $list ]; then " +
|
|
"echo PRESERVE CAPTURED NOTHING BUT AN EARLIER ARCHIVE HAS FILES; exit 8; " +
|
|
"fi; " +
|
|
"done; " +
|
|
"fi; " +
|
|
"ls -1t ${glob}.tgz 2>/dev/null | tail -n +$drop | xargs -r rm -f; " +
|
|
"ls -1t ${glob}.list 2>/dev/null | tail -n +$drop | xargs -r rm -f; " +
|
|
"exit 0"
|
|
|
|
Invoke-Ec2Step "preserve operator files ($spec)" $bash `
|
|
-FailHint "Refusing to wipe $RemotePath - see $glob.tgz on the server."
|
|
}
|
|
|
|
function Restore-OperatorFiles {
|
|
param([string]$Key, [string]$RemotePath)
|
|
$tarball = Get-PreserveTarball -Key $Key
|
|
# Overwrites, deliberately: server-side operator files beat whatever the
|
|
# zip shipped. The archive is left in place - see the header.
|
|
Invoke-Ec2Step "restore operator files" "test -f $tarball && tar -xzf $tarball -C $RemotePath; true"
|
|
}
|
|
|
|
# ── Deploy verification (build-version match, not just HTTP 200 — a 200 can be
|
|
# a stale cached build; the version match proves the new build is live) ─────
|
|
|
|
function Wait-VerifyStaticBuild {
|
|
param([string]$Key, $Proj, [object]$PreZipBuildState)
|
|
if (-not $PreZipBuildState) {
|
|
Write-Host "`n--- [$Key version] SKIPPED (no local build-version.json - see 'Enabling deploy verification' in README) ---" -ForegroundColor DarkYellow
|
|
return
|
|
}
|
|
# Expect the COMMITTED stamp, not +1: builds no longer self-bump (a
|
|
# prebuild hook incremented inside the image, so the served version
|
|
# matched no commit and every build dirtied the tree). The counter now
|
|
# advances deliberately — one version bump per release — so "is the
|
|
# build I just packed live?" means an exact match.
|
|
$expectedLabel = Get-LabelFromBuildJsonObj $PreZipBuildState
|
|
Write-Host "`n--- [$Key] Live build verification (expect $expectedLabel) ---" -ForegroundColor Cyan
|
|
$containerName = $Proj.remote.containerName
|
|
$deadline = (Get-Date).AddSeconds(45)
|
|
while ((Get-Date) -lt $deadline) {
|
|
try {
|
|
$r = $null
|
|
if ($containerName) {
|
|
# build-version.json may be blocked from external requests by the edge
|
|
# proxy; read it inside the running container instead.
|
|
$raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET `
|
|
"sudo docker exec $containerName cat /usr/share/nginx/html/build-version.json 2>/dev/null"
|
|
if ($raw) { $r = $raw | ConvertFrom-Json -ErrorAction Stop }
|
|
} else {
|
|
$headers = @{}
|
|
if ($Proj.domain) { $headers['Host'] = $Proj.domain }
|
|
$r = Invoke-RestMethod -Uri "http://$EC2_IP/build-version.json" -Headers $headers -TimeoutSec 10 -ErrorAction Stop
|
|
}
|
|
if ($r) {
|
|
$remoteLabel = Get-LabelFromBuildJsonObj $r
|
|
if ($remoteLabel -eq $expectedLabel) {
|
|
Write-Host " PASS - live build $remoteLabel matches expected." -ForegroundColor Green
|
|
return
|
|
}
|
|
Write-Host " Live build is $remoteLabel, expected $expectedLabel - waiting..." -ForegroundColor DarkYellow
|
|
}
|
|
} catch {
|
|
Write-Host " Container not ready yet - waiting..." -ForegroundColor DarkGray
|
|
}
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
Write-Host " WARNING: live build did not match $expectedLabel within 45s (upload or Docker build may have failed, or a stale build is cached)." -ForegroundColor Yellow
|
|
}
|
|
|
|
function Wait-VerifyApiBuild {
|
|
param([string]$Key, $Proj, [string]$ExpectedLabel, [int]$TimeoutSec = 60)
|
|
Write-Host "`n--- [$Key] Live build verification (expect $ExpectedLabel) ---" -ForegroundColor Cyan
|
|
# deploy.verifyHost (or domain) decides which edge vhost may be asked;
|
|
# both are consumed inside Get-VerifyAttempts now, where "no host at all"
|
|
# excludes the edge channel entirely rather than defaulting to whatever
|
|
# vhost the proxy serves (#101). verifyHost exists for a domain retired
|
|
# ahead of its replacement - a takedown once had a project's public host
|
|
# answering 410 while the app was healthy; no project sets it today.
|
|
# Every retry walks the channels in trust order - docker-network exec,
|
|
# then localhost port, then (only with a Host to route by) the edge.
|
|
# The choice used to be made ONCE, before the loop, by probing each
|
|
# channel - but the probes ran at the exact moment step [5] had
|
|
# restarted the app, so both good channels were briefly down and the
|
|
# whole window was spent on the edge. For a project with no domain that
|
|
# meant the default vhost: one project's check read a DIFFERENT
|
|
# project's build number, twice in a single day (#101). Re-resolving per
|
|
# retry means the right channel is used the moment the app is back.
|
|
#
|
|
# The port channel still counts only when the body carries a version:
|
|
# one project's verify path is a JWKS endpoint - real, healthy, and no
|
|
# version in it - so it falls through to the edge (it has a domain),
|
|
# same as it always did.
|
|
$execCmd = Get-ServerSideVersionCommand -Proj $Proj
|
|
$attempts = Get-VerifyAttempts -Proj $Proj -ExecCmd $execCmd
|
|
$TimeoutSec = Get-VerifyTimeout -Proj $Proj -DefaultSec $TimeoutSec
|
|
if ($attempts.Count -eq 0) {
|
|
# No trustworthy channel exists: no viaProxy, no port, no host to
|
|
# route an edge request by. Asking the edge anyway can only reach
|
|
# the DEFAULT vhost - a different product - and a check that can
|
|
# only ever read someone else's number is worse than no check.
|
|
Write-Host " SKIPPED: no way to verify this project without reading the wrong vhost - configure verify.viaProxy/port, or a domain (#101)." -ForegroundColor Yellow
|
|
return $false
|
|
}
|
|
Write-Host " Channels, in order: $(($attempts | ForEach-Object { $_.Label }) -join '; ')" -ForegroundColor DarkGray
|
|
|
|
$sawVersion = $false
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
while ((Get-Date) -lt $deadline) {
|
|
foreach ($attempt in $attempts) {
|
|
$r = $null
|
|
try {
|
|
switch ($attempt.Kind) {
|
|
'exec' {
|
|
$raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $execCmd
|
|
$r = ($raw | Out-String).Trim() | ConvertFrom-Json -ErrorAction Stop
|
|
}
|
|
'port' {
|
|
$raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "curl -s -m 8 http://localhost:$($attempt.Port)$($attempt.Path)"
|
|
$r = ($raw | Out-String).Trim() | ConvertFrom-Json -ErrorAction Stop
|
|
}
|
|
'edge' {
|
|
$edgeHeaders = @{ 'Host' = $attempt.HostHeader }
|
|
$r = Invoke-RestMethod -Uri "http://$EC2_IP/api/build-version" -Headers $edgeHeaders -TimeoutSec 10 -ErrorAction Stop
|
|
}
|
|
}
|
|
} catch {
|
|
continue # channel not ready; the next one gets its turn
|
|
}
|
|
if ($null -eq $r) { continue }
|
|
# Two field names in the fleet: some apps answer build_version
|
|
# on /api/build-version, others answer version on /health. Both
|
|
# are "the build that is live", so accept either rather than
|
|
# making every app rename its own field.
|
|
$live = if ($r.build_version) { [string]$r.build_version } elseif ($r.version) { [string]$r.version } else { $null }
|
|
if (-not $live) { continue } # answered, but not about versions (JWKS etc.)
|
|
$sawVersion = $true
|
|
if ($live -eq $ExpectedLabel) {
|
|
Write-Host " PASS - live build $live matches expected ($($attempt.Label))." -ForegroundColor Green
|
|
return $true
|
|
}
|
|
Write-Host " Live build is $live via $($attempt.Label), expected $ExpectedLabel - waiting..." -ForegroundColor DarkYellow
|
|
break # one wrong-version read this pass is enough; retry after the sleep
|
|
}
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
# Say which failure this actually was: a version that never matched is a
|
|
# stale/failed build; channels that never answered is "could not verify",
|
|
# and pretending otherwise is how a warning gets ignored.
|
|
if ($sawVersion) {
|
|
Write-Host " WARNING: live build did not match $ExpectedLabel within ${TimeoutSec}s (upload or Docker build may have failed, or a stale build is cached)." -ForegroundColor Yellow
|
|
} else {
|
|
Write-Host " WARNING: could not verify within ${TimeoutSec}s - no channel answered with a version (app may still be starting; raise verify.timeoutSeconds if this project boots slowly)." -ForegroundColor Yellow
|
|
}
|
|
return $false
|
|
}
|
|
|
|
# Verify a deploy by calling the app ON the server (localhost:<port>). Works
|
|
# for stacks that are not published through the edge proxy or whose host port
|
|
# is closed to the internet — hitting http://<ec2-ip>/ for those just answers
|
|
# from whatever vhost the proxy serves by default, which is a false PASS.
|
|
#
|
|
# Configure per project in zconfig.json:
|
|
# "verify": { "port": 8005, "path": "/health", "expect": "\"status\":\"ok\"" }
|
|
# port is required; path defaults to "/", expect is an optional substring.
|
|
function Test-DeployHealth {
|
|
param([string]$Key, $Proj, [int]$TimeoutSec = 60)
|
|
|
|
$port = [int]$Proj.verify.port
|
|
$path = if ($Proj.verify.path) { [string]$Proj.verify.path } else { "/" }
|
|
$expect = [string]$Proj.verify.expect
|
|
|
|
Write-Host "`n--- [$Key] Health check (on server: localhost:$port$path) ---" -ForegroundColor Cyan
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
while ((Get-Date) -lt $deadline) {
|
|
# -L: an app whose "/" redirects (e.g. Next.js "/" -> "/login") answers a
|
|
# 307 whose body is a few bytes or empty, which reads as "not ready".
|
|
# Follow to the page that actually renders before judging.
|
|
$raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "curl -sL -m 8 http://localhost:$port$path"
|
|
$body = ($raw | Out-String).Trim()
|
|
if ($LASTEXITCODE -eq 0 -and $body) {
|
|
if (-not $expect -or $body.Contains($expect)) {
|
|
# A /health endpoint returns a line of JSON; an app page returns
|
|
# kilobytes of HTML. Match on the whole body, but only print
|
|
# enough to recognise it — the rest is unreadable in a log.
|
|
$shown = if ($body.Length -gt 200) { $body.Substring(0, 200) + "... ($($body.Length) bytes)" } else { $body }
|
|
Write-Host " PASS - $shown" -ForegroundColor Green
|
|
return $true
|
|
}
|
|
Write-Host " Responding but '$expect' not found - waiting..." -ForegroundColor DarkYellow
|
|
} else {
|
|
Write-Host " Not ready yet - waiting..." -ForegroundColor DarkGray
|
|
}
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
Write-Host " WARNING: no healthy response from localhost:$port$path within ${TimeoutSec}s." -ForegroundColor Yellow
|
|
return $false
|
|
}
|
|
|
|
# Where the thing just deployed can be reached. A project published through the
|
|
# edge proxy has a domain; one that is not still has somewhere to point at, and
|
|
# saying nothing is the least useful option — an internal service is exactly the
|
|
# case where "where did that land?" is hardest to answer from memory. Falls back
|
|
# through what the project actually declares, and prints nothing if it declares
|
|
# none of it.
|
|
function Write-DeployLocation {
|
|
param($Proj, [int]$Pad = 0)
|
|
$label = "Site:".PadRight([Math]::Max(5, $Pad))
|
|
if ($Proj.domain) {
|
|
Write-Host "$label https://$($Proj.domain)" -ForegroundColor Yellow
|
|
return
|
|
}
|
|
# No public route: give the server-local endpoint the deploy just verified.
|
|
$port = if ($Proj.verify -and $Proj.verify.port) { [int]$Proj.verify.port }
|
|
elseif ($Proj.ports -and $Proj.ports.prod) { [int]$Proj.ports.prod }
|
|
else { 0 }
|
|
if ($port -le 0) { return }
|
|
$path = if ($Proj.verify -and $Proj.verify.path) { [string]$Proj.verify.path } else { "" }
|
|
Write-Host "$label http://127.0.0.1:$port$path (on the server; no public domain)" -ForegroundColor Yellow
|
|
}
|
|
|
|
# ── Kind handlers ────────────────────────────────────────────────────────────
|
|
|
|
function Invoke-PythonDeploy {
|
|
param([string]$Key, $Proj, [string]$ChangeNote)
|
|
|
|
$DeployStart = Get-Date
|
|
$prevLoc = Get-Location
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
$composeDir = if ($Proj.remote.composeDir) { $Proj.remote.composeDir } else { $remotePath }
|
|
$appSvc = if ($Proj.remote.appService) { $Proj.remote.appService } else { "app" }
|
|
$zipName = Get-DeployZipName -Key $Key -Proj $Proj
|
|
$zipLocal = Join-Path $TempRoot $zipName
|
|
$BuildVersion = $null
|
|
$versionTool = Join-Path $root "scripts\build_version_tool.py"
|
|
$hasVersionTool = Test-Path -LiteralPath $versionTool
|
|
|
|
try {
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
|
|
Set-Location -LiteralPath $root
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (python) ===" -ForegroundColor Cyan
|
|
Write-Host "Local zip: $zipLocal" -ForegroundColor DarkGray
|
|
|
|
Write-Host "`n--- [1] Zipping $($Proj.label) ---" -ForegroundColor Cyan
|
|
Get-ChildItem -LiteralPath $root -Directory -Recurse -Filter "__pycache__" -ErrorAction SilentlyContinue |
|
|
Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
|
|
New-ProjectArchive -SourcePath $root -DestinationZip $zipLocal -TopLevelExclude (Get-ArchiveExcludes -Project $Proj)
|
|
|
|
Invoke-Ec2PreflightCleanup -ExtraZipsToRemove @("$RemoteHome/$zipName")
|
|
|
|
Write-Host "`n--- [2] Uploading zip ---" -ForegroundColor Cyan
|
|
Send-DeployZip -LocalZip $zipLocal -ZipName $zipName
|
|
|
|
Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
|
|
Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
|
|
Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
|
|
Invoke-Ec2Step "replace project directory" "sudo rm -rf $remotePath && sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
|
|
Invoke-RemoteUnzip -ZipName $zipName -DestPath $remotePath
|
|
Restore-OperatorFiles -Key $Key -RemotePath $remotePath
|
|
Invoke-Ec2Step "require compose directory" "test -d $composeDir"
|
|
Invoke-Ec2Step "docker compose build $appSvc" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose build $appSvc"
|
|
Invoke-Ec2Step "docker compose up -d" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose up -d"
|
|
Invoke-Ec2Step "record deploy time; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remotePath/.last_deploy_utc > /dev/null && rm -f $RemoteHome/$zipName"
|
|
|
|
if ($hasVersionTool) {
|
|
Write-Host "`n--- [4] Incrementing build version ---" -ForegroundColor Cyan
|
|
$BumpCmd = "cd $composeDir && sudo docker compose exec -T $appSvc python scripts/build_version_tool.py bump"
|
|
for ($attempt = 1; $attempt -le 5; $attempt++) {
|
|
$output = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $BumpCmd
|
|
if ($LASTEXITCODE -eq 0 -and $output) {
|
|
$BuildVersion = ($output | Select-Object -Last 1).ToString().Trim()
|
|
break
|
|
}
|
|
Write-Host " Attempt $attempt failed, retrying in 3s..." -ForegroundColor DarkYellow
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
if (-not $BuildVersion) { throw "Build version bump failed after 5 attempts" }
|
|
|
|
# The local stamp is deliberately NOT mirrored back. Writing it
|
|
# left build-version.json dirty after every deploy, and committing
|
|
# that hit branch protection ("Changes must be made through a pull
|
|
# request") — so each deploy either tripped the NEXT deploy's
|
|
# clean-tree guard or bypassed the rule. Neither is acceptable as
|
|
# routine behaviour.
|
|
#
|
|
# The repo file now records the STAGE baseline only (it changes on
|
|
# a stage bump, through a normal PR). The live build number lives
|
|
# in the container, is written to .build_version below, and is
|
|
# proven by the /api/build-version check — which is the thing that
|
|
# actually establishes what is deployed.
|
|
Invoke-Ec2Step "record the live build number" "echo '$BuildVersion' | sudo tee $remotePath/.build_version > /dev/null"
|
|
$changelogTool = Join-Path $root "scripts\build_changelog_tool.py"
|
|
if (Test-Path -LiteralPath $changelogTool) {
|
|
if ([string]::IsNullOrWhiteSpace($ChangeNote)) { $ChangeNote = "Build deployed" }
|
|
python $changelogTool append --version $BuildVersion --note "$ChangeNote" | Out-Null
|
|
}
|
|
|
|
Write-Host "`n--- [5] Restarting app to pick up new version ---" -ForegroundColor Cyan
|
|
# Through Invoke-Ec2Step, not a bare ssh: `docker compose restart`
|
|
# writes " Container <name> Restarting" to STDERR as ordinary
|
|
# progress, and under ErrorActionPreference='Stop' PS 5.1 turns any
|
|
# native stderr line into a terminating NativeCommandError whatever
|
|
# the exit code. That threw here on a deploy that had fully
|
|
# succeeded - and it threw BEFORE Wait-VerifyApiBuild, so the step
|
|
# that proves what is actually deployed never ran (#119). The
|
|
# wrapper flattens stderr and judges by exit code alone, and throws
|
|
# on non-zero itself, so the hand-written check is gone with it.
|
|
Invoke-Ec2Step "restart $appSvc to pick up the new build" `
|
|
"cd $composeDir && sudo COMPOSE_BAKE=false docker compose restart $appSvc" `
|
|
-FailHint "App restart after the build bump failed."
|
|
|
|
Wait-VerifyApiBuild -Key $Key -Proj $Proj -ExpectedLabel $BuildVersion -TimeoutSec 30 | Out-Null
|
|
} elseif ($Proj.verify -and $Proj.verify.port) {
|
|
Test-DeployHealth -Key $Key -Proj $Proj -TimeoutSec 60 | Out-Null
|
|
} elseif ($Proj.domain) {
|
|
Write-Host "`n--- [4] Basic reachability check (no build_version_tool - see 'Enabling deploy verification' in README) ---" -ForegroundColor Cyan
|
|
$headers = @{ 'Host' = $Proj.domain }
|
|
$deadline = (Get-Date).AddSeconds(30)
|
|
$up = $false
|
|
while ((Get-Date) -lt $deadline) {
|
|
Start-Sleep -Seconds 3
|
|
try {
|
|
$resp = Invoke-WebRequest -Uri "http://$EC2_IP/" -Headers $headers -UseBasicParsing -TimeoutSec 8 -ErrorAction Stop
|
|
if ($resp.StatusCode -lt 500) { $up = $true; break }
|
|
} catch { Write-Host " App not ready yet - waiting..." -ForegroundColor DarkGray }
|
|
}
|
|
if ($up) { Write-Host " App is responding." -ForegroundColor Green }
|
|
else { Write-Host " WARNING: app did not respond within 30s." -ForegroundColor Yellow }
|
|
} else {
|
|
# No domain to send as a Host header and no "verify" block: a request
|
|
# to http://<ec2-ip>/ would be answered by the proxy's default vhost,
|
|
# so it proves nothing about THIS app. Say so instead of faking a PASS.
|
|
Write-Host "`n--- [4] Deploy finished - NOT verified ---" -ForegroundColor Yellow
|
|
Write-Host " No 'domain' and no 'verify' block in zconfig.json for '$Key'," -ForegroundColor Yellow
|
|
Write-Host " so there is no way to confirm the new build is live." -ForegroundColor Yellow
|
|
Write-Host ' Add to the project: "verify": { "port": <hostPort>, "path": "/health" }' -ForegroundColor Gray
|
|
}
|
|
|
|
Invoke-Ec2PostDeployCleanup -Label $Key
|
|
|
|
$Elapsed = (Get-Date) - $DeployStart
|
|
$ElapsedStr = "{0:mm\:ss}" -f $Elapsed
|
|
Write-Host "`n--- [Done] $($Proj.label) deployed! ---" -ForegroundColor Green
|
|
Write-DeployLocation -Proj $Proj
|
|
if ($BuildVersion) { Write-Host "Build Version: $BuildVersion" -ForegroundColor Magenta }
|
|
Write-Host "Change Note: $ChangeNote" -ForegroundColor Cyan
|
|
Write-Host "Deploy Time: $ElapsedStr ($([math]::Round($Elapsed.TotalSeconds))s)" -ForegroundColor DarkGray
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $zipLocal) {
|
|
try { Remove-Item -LiteralPath $zipLocal -Force -ErrorAction Stop } catch { }
|
|
}
|
|
Set-Location -LiteralPath $prevLoc
|
|
}
|
|
}
|
|
|
|
function Invoke-ViteDeploy {
|
|
param([string]$Key, $Proj, [string]$ChangeNote)
|
|
|
|
$DeployStart = Get-Date
|
|
$prevLoc = Get-Location
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
$zipName = Get-DeployZipName -Key $Key -Proj $Proj
|
|
$zipLocal = Join-Path $TempRoot $zipName
|
|
$preZipBuild = $null
|
|
|
|
try {
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
|
|
Set-Location -LiteralPath $root
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (vite/static) ===" -ForegroundColor Cyan
|
|
Write-Host "Local zip: $zipLocal" -ForegroundColor DarkGray
|
|
|
|
Write-Host "`n--- [1] Zipping site ---" -ForegroundColor Cyan
|
|
$preZipBuild = Read-JsonBuildVersion -FilePath (Join-Path $root "build-version.json")
|
|
if ($preZipBuild) {
|
|
Write-Host " Pre-zip build label: $(Get-LabelFromBuildJsonObj $preZipBuild) (verification expects this exact label)" -ForegroundColor Gray
|
|
}
|
|
New-ProjectArchive -SourcePath $root -DestinationZip $zipLocal -TopLevelExclude (Get-ArchiveExcludes -Project $Proj)
|
|
|
|
Invoke-Ec2PreflightCleanup -ExtraZipsToRemove @("$RemoteHome/$zipName")
|
|
|
|
Write-Host "`n--- [2] Uploading zip ---" -ForegroundColor Cyan
|
|
Send-DeployZip -LocalZip $zipLocal -ZipName $zipName
|
|
|
|
Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
|
|
Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
|
|
Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
|
|
Invoke-Ec2Step "replace project directory" "sudo rm -rf $remotePath && sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
|
|
Invoke-RemoteUnzip -ZipName $zipName -DestPath $remotePath
|
|
Restore-OperatorFiles -Key $Key -RemotePath $remotePath
|
|
Invoke-Ec2Step "require compose file" "test -f $remotePath/docker-compose.yml"
|
|
Invoke-Ec2Step "docker compose build" "cd $remotePath && sudo COMPOSE_BAKE=false docker compose build"
|
|
Invoke-Ec2Step "docker compose up -d" "cd $remotePath && sudo COMPOSE_BAKE=false docker compose up -d"
|
|
|
|
$edgeProj = Get-ZEdgeProject
|
|
if ($edgeProj -and $edgeProj.Config.proxyContainer) {
|
|
Invoke-Ec2Step "reload edge nginx (flush DNS cache for new container IP)" "sudo docker exec $($edgeProj.Config.proxyContainer) nginx -s reload"
|
|
}
|
|
Invoke-Ec2Step "record deploy time; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remotePath/.last_deploy_utc > /dev/null && rm -f $RemoteHome/$zipName"
|
|
|
|
Wait-VerifyStaticBuild -Key $Key -Proj $Proj -PreZipBuildState $preZipBuild
|
|
Invoke-Ec2PostDeployCleanup -Label $Key
|
|
|
|
$Elapsed = (Get-Date) - $DeployStart
|
|
$ElapsedStr = "{0:mm\:ss}" -f $Elapsed
|
|
Write-Host "`n--- [Done] $($Proj.label) deployed! ---" -ForegroundColor Green
|
|
Write-DeployLocation -Proj $Proj
|
|
Write-Host "Change Note: $ChangeNote" -ForegroundColor Cyan
|
|
Write-Host "Deploy Time: $ElapsedStr ($([math]::Round($Elapsed.TotalSeconds))s)" -ForegroundColor DarkGray
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $zipLocal) {
|
|
try { Remove-Item -LiteralPath $zipLocal -Force -ErrorAction Stop } catch { }
|
|
}
|
|
Set-Location -LiteralPath $prevLoc
|
|
}
|
|
}
|
|
|
|
function Invoke-NextDeploy {
|
|
param([string]$Key, $Proj, [string]$ChangeNote)
|
|
|
|
$DeployStart = Get-Date
|
|
$prevLoc = Get-Location
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
# Same resolution as the python handler. Must be computed HERE: PowerShell
|
|
# function scope means the copy in Invoke-PythonDeploy is invisible from
|
|
# this one, and an unset variable interpolates to an empty string — so
|
|
# "cd && docker compose down" quietly runs in the home directory.
|
|
$composeDir = if ($Proj.remote.composeDir) { $Proj.remote.composeDir } else { $remotePath }
|
|
$appSvc = if ($Proj.remote.appService) { $Proj.remote.appService } else { "web" }
|
|
$zipName = Get-DeployZipName -Key $Key -Proj $Proj
|
|
$zipLocal = Join-Path $TempRoot $zipName
|
|
$preZipBuild = $null
|
|
|
|
try {
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
|
|
Set-Location -LiteralPath $root
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (nextjs) ===" -ForegroundColor Cyan
|
|
Write-Host "Local zip: $zipLocal" -ForegroundColor DarkGray
|
|
|
|
# Stamp the build version from git before zipping, if the project says
|
|
# how (deploy.stampCmd in zconfig). The image has no .git - it is in the
|
|
# archive excludes - so the number has to be written on this side of the
|
|
# zip.
|
|
#
|
|
# Written, zipped, then reverted: zdeploy refuses a dirty tree, so a
|
|
# stamp that dirtied it every deploy would block the next one. The
|
|
# committed file stays a fallback for local dev; the number that ships
|
|
# is derived from the commit being deployed.
|
|
$stampCmd = if ($Proj.deploy -and $Proj.deploy.stampCmd) { $Proj.deploy.stampCmd } else { $null }
|
|
$stampFile = if ($Proj.deploy -and $Proj.deploy.stampFile) { $Proj.deploy.stampFile } else { "public/build-version.json" }
|
|
$stampedLabel = $null
|
|
if ($stampCmd) {
|
|
Write-Host "`n--- [0] Stamping build version from git ---" -ForegroundColor Cyan
|
|
$out = & cmd /c $stampCmd 2>&1
|
|
if ($LASTEXITCODE -ne 0) { throw "Build stamp failed: $out" }
|
|
$stampedLabel = ($out | Select-Object -Last 1).ToString().Trim()
|
|
Write-Host " $stampedLabel (derived from the commit, not a counter)" -ForegroundColor Gray
|
|
}
|
|
|
|
$preZipBuild = Read-JsonBuildVersion -FilePath (Join-Path $root "public\build-version.json")
|
|
if ($preZipBuild -and -not $stampedLabel) {
|
|
Write-Host " Pre-zip build label: $(Get-LabelFromBuildJsonObj $preZipBuild)" -ForegroundColor Gray
|
|
}
|
|
|
|
Write-Host "`n--- [1] Zipping project files ---" -ForegroundColor Cyan
|
|
New-ProjectArchive -SourcePath $root -DestinationZip $zipLocal -TopLevelExclude (Get-ArchiveExcludes -Project $Proj)
|
|
|
|
if ($stampCmd) {
|
|
# Tree back to clean now the number is inside the archive.
|
|
git -C $root checkout -- $stampFile 2>&1 | Out-Null
|
|
}
|
|
|
|
Invoke-Ec2PreflightCleanup -ExtraZipsToRemove @("$RemoteHome/$zipName")
|
|
|
|
Write-Host "`n--- [2] Uploading zip ---" -ForegroundColor Cyan
|
|
Send-DeployZip -LocalZip $zipLocal -ZipName $zipName
|
|
|
|
Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
|
|
Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
|
|
Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
|
|
Invoke-Ec2Step "replace project directory" "sudo rm -rf $remotePath && sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
|
|
Invoke-RemoteUnzip -ZipName $zipName -DestPath $remotePath
|
|
Restore-OperatorFiles -Key $Key -RemotePath $remotePath
|
|
|
|
Write-Host "`n--- [4] Docker compose rebuild ---" -ForegroundColor Cyan
|
|
# composeDir, not remotePath: a project whose compose lives in a
|
|
# subdirectory (deploy/, infra/, ...) otherwise runs these against
|
|
# whatever docker-compose.yml happens to sit at the project root. That
|
|
# is usually the LOCAL DEV compose, which ships in the same archive —
|
|
# so the deploy recycled a dev database, found no app service to build,
|
|
# exited 0, and left the previous image serving. Verification passed
|
|
# because the untouched old container still answered. Two deploys in a
|
|
# row silently shipped nothing.
|
|
# Assert locally, not just on the server: `test -d $composeDir` with an
|
|
# empty value becomes bare `test -d`, which is TRUE (one non-empty
|
|
# argument), so the remote guard cannot catch this.
|
|
if ([string]::IsNullOrWhiteSpace($composeDir)) { throw "composeDir resolved empty for '$Key' - compose would run in the wrong directory." }
|
|
Invoke-Ec2Step "require compose file" "test -f $composeDir/docker-compose.yml || test -f $composeDir/docker-compose.yaml"
|
|
# BUILD BEFORE DOWN. This used to run `down` first, which took the site
|
|
# offline for the whole build - minutes for a Next.js app - and left it
|
|
# offline if the build failed. That is not hypothetical: one deploy
|
|
# stopped the stack, the build did not finish, and the site served 502
|
|
# for 19 hours with no container at all. The
|
|
# old image keeps serving while the new one builds, so a failed build is
|
|
# now harmless and the outage is the seconds between down and up.
|
|
#
|
|
# Named service, like the python handler: a compose file that does not
|
|
# define it fails here instead of succeeding with nothing to do.
|
|
Invoke-Ec2Step "docker compose build $appSvc" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose build $appSvc"
|
|
Invoke-Ec2Step "docker compose down" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose down"
|
|
Invoke-Ec2Step "docker compose up -d" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose up -d"
|
|
|
|
if ($Proj.db -and $Proj.db.user -and $Proj.db.name) {
|
|
$waitDb = "cd $composeDir && for i in `$(seq 1 30); do sudo docker compose exec -T db pg_isready -U $($Proj.db.user) -d $($Proj.db.name) >/dev/null 2>&1 && break; sleep 2; done"
|
|
Invoke-Ec2Step "wait for postgres ready" $waitDb
|
|
}
|
|
if ($Proj.migrations -eq "prisma") {
|
|
Invoke-Ec2Step "apply prisma migrations" "cd $composeDir && sudo docker compose exec -T $appSvc npx prisma migrate deploy"
|
|
}
|
|
Invoke-Ec2Step "record deploy timestamp; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remotePath/.last_deploy_utc > /dev/null && rm -f $RemoteHome/$zipName"
|
|
|
|
Write-Host "`n--- [5] Verifying deployment ---" -ForegroundColor Cyan
|
|
# Same precedence as the python handler. Do NOT probe http://<ec2-ip>:<prod-port>/
|
|
# here: a compose stack behind the edge proxy usually publishes to
|
|
# 127.0.0.1 only, so that probe can never answer and the old "is the port
|
|
# open in the security group?" warning sent you chasing a firewall rule
|
|
# for an app that was already up. See issue #28.
|
|
if ($Proj.verify -and $Proj.verify.port) {
|
|
Test-DeployHealth -Key $Key -Proj $Proj -TimeoutSec 60 | Out-Null
|
|
} elseif ($Proj.domain) {
|
|
$headers = @{ 'Host' = $Proj.domain }
|
|
$deadline = (Get-Date).AddSeconds(60)
|
|
$verified = $false
|
|
while ((Get-Date) -lt $deadline) {
|
|
Start-Sleep -Seconds 4
|
|
try {
|
|
$resp = Invoke-WebRequest -Uri "http://$EC2_IP/" -Headers $headers -TimeoutSec 8 -ErrorAction Stop -UseBasicParsing
|
|
if ($resp.StatusCode -lt 500) {
|
|
Write-Host " PASS - app is responding at https://$($Proj.domain)/" -ForegroundColor Green
|
|
$verified = $true
|
|
break
|
|
}
|
|
} catch {
|
|
Write-Host " App not ready yet - waiting..." -ForegroundColor DarkGray
|
|
}
|
|
}
|
|
if (-not $verified) {
|
|
Write-Host " WARNING: no response for https://$($Proj.domain)/ within 60s." -ForegroundColor Yellow
|
|
}
|
|
} else {
|
|
Write-Host " Deploy finished - NOT verified." -ForegroundColor Yellow
|
|
Write-Host " No 'domain' and no 'verify' block in zconfig.json for '$Key'." -ForegroundColor Yellow
|
|
}
|
|
if ($stampedLabel -or $preZipBuild) {
|
|
# The label that actually went into the archive: the derived one
|
|
# when the project stamps, otherwise the committed stamp.
|
|
$expectedLabel = if ($stampedLabel) { $stampedLabel } else { Get-LabelFromBuildJsonObj $preZipBuild }
|
|
Wait-VerifyApiBuild -Key $Key -Proj $Proj -ExpectedLabel $expectedLabel -TimeoutSec 60 | Out-Null
|
|
} else {
|
|
Write-Host " (No public/build-version.json - version verification skipped. See 'Enabling deploy verification' in README.)" -ForegroundColor DarkYellow
|
|
}
|
|
|
|
Invoke-Ec2PostDeployCleanup -Label $Key
|
|
|
|
$Elapsed = (Get-Date) - $DeployStart
|
|
$ElapsedStr = "{0:mm\:ss}" -f $Elapsed
|
|
Write-Host "`n--- [Done] $($Proj.label) deployed! ---" -ForegroundColor Green
|
|
Write-DeployLocation -Proj $Proj -Pad 12
|
|
Write-Host "Change Note: $ChangeNote" -ForegroundColor Cyan
|
|
Write-Host "Deploy Time: $ElapsedStr ($([math]::Round($Elapsed.TotalSeconds))s)" -ForegroundColor DarkGray
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $zipLocal) {
|
|
try { Remove-Item -LiteralPath $zipLocal -Force -ErrorAction Stop } catch { }
|
|
}
|
|
Set-Location -LiteralPath $prevLoc
|
|
}
|
|
}
|
|
|
|
function Invoke-EdgeDeploy {
|
|
param([string]$Key, $Proj)
|
|
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
$pc = $Proj.proxyContainer
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (edge nginx ingress) ===" -ForegroundColor Cyan
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Edge root not found: $root" }
|
|
foreach ($required in @('docker-compose.yml', 'nginx.conf')) {
|
|
if (-not (Test-Path -LiteralPath (Join-Path $root $required))) { throw "Missing $root\$required" }
|
|
}
|
|
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
# -R: docker creates mount-point subdirs (vendor/, fonts/) root-owned when
|
|
# they are missing at compose up; a non-recursive chown leaves those
|
|
# unwritable and every scp into them fails.
|
|
Invoke-Ec2Step "ensure edge dir" "sudo mkdir -p $remotePath && sudo chown -R ${Ec2User}:${Ec2User} $remotePath"
|
|
|
|
# Ship every top-level file in the edge folder — nginx.conf, compose, css,
|
|
# htpasswd, whatever the proxy serves.
|
|
$files = @(Get-ChildItem -LiteralPath $root -File | Where-Object { $_.Name -ne 'nul' })
|
|
foreach ($f in $files) {
|
|
Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan
|
|
scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
# Content subdirectories the proxy serves (fonts/, vendor/, ...) ship too —
|
|
# only server-side state stays put. Skipping them is how the self-hosted
|
|
# Chart.js and fonts silently never reached prod (charts rendered blank).
|
|
# .pytest_cache is a local test artifact, already gitignored; it has no
|
|
# business on the proxy box and only adds noise to the upload log.
|
|
$skipDirs = @('nginx-logs', '.git', '.pytest_cache')
|
|
$dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
|
|
foreach ($d in $dirs) {
|
|
Write-Host " >> uploading $($d.Name)/ (recursive)" -ForegroundColor DarkCyan
|
|
scp -r @SCP_OPTS -i $PEM_KEY $d.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
$certMount = if ($Proj.certsSource) { "-v $($Proj.certsSource):/etc/letsencrypt/:ro " } else { "" }
|
|
Invoke-Ec2Step "validate new nginx.conf" "sudo docker run --rm -v $remotePath/nginx.conf:/etc/nginx/nginx.conf:ro ${certMount}nginx:1.27-alpine nginx -t -c /etc/nginx/nginx.conf"
|
|
|
|
# A container from an older compose project may still hold the proxy name;
|
|
# docker refuses a second create with the same name, so remove it first.
|
|
$rmStale = if ($pc) { "; sudo docker rm -f $pc 2>/dev/null || true" } else { "" }
|
|
Invoke-Ec2Step "edge: compose down + remove stale proxy" "cd $remotePath && sudo docker compose down 2>/dev/null || true$rmStale"
|
|
Invoke-Ec2Step "edge compose up -d" "cd $remotePath && sudo docker compose up -d"
|
|
if ($pc) {
|
|
Invoke-Ec2Step "edge nginx reload" "sudo docker exec $pc nginx -s reload || true"
|
|
}
|
|
Invoke-Ec2Step "fix nginx-logs permissions (if present)" "if [ -d $remotePath/nginx-logs ]; then sudo chmod 777 $remotePath/nginx-logs; sudo chmod 666 $remotePath/nginx-logs/*.log 2>/dev/null || true; fi"
|
|
Write-Host "--- [Done] Edge proxy deploy finished ---" -ForegroundColor Green
|
|
}
|
|
|
|
function Invoke-StaticDeploy {
|
|
param([string]$Key, $Proj)
|
|
|
|
# Plain static sites - no build, no container of their own. The landing
|
|
# container serves them straight off disk out of /srv/$host, so shipping
|
|
# the files IS the deploy: there is nothing to restart afterwards.
|
|
$root = Join-Path $Proj.localRoot $Proj.siteDir
|
|
$remotePath = $Proj.remote.path
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (static files) ===" -ForegroundColor Cyan
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Static site root not found: $root" }
|
|
if (-not (Test-Path -LiteralPath (Join-Path $root 'index.html'))) { throw "Missing $root\index.html" }
|
|
|
|
Invoke-Ec2Step "ensure site dir" "sudo mkdir -p $remotePath && sudo chown -R ${Ec2User}:${Ec2User} $remotePath"
|
|
|
|
$files = @(Get-ChildItem -LiteralPath $root -File)
|
|
foreach ($f in $files) {
|
|
Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan
|
|
scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
# A large media file uploaded in place is served half-written to anyone who
|
|
# requests it mid-copy. Ship the directory to a sibling, then swap it in.
|
|
$skipDirs = @('.git', '.pytest_cache', 'node_modules')
|
|
$dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
|
|
foreach ($d in $dirs) {
|
|
Write-Host " >> uploading $($d.Name)/ (recursive, staged)" -ForegroundColor DarkCyan
|
|
Invoke-Ec2Step "stage $($d.Name)" "rm -rf $remotePath/.staging-$($d.Name) && mkdir -p $remotePath/.staging-$($d.Name)"
|
|
scp -r @SCP_OPTS -i $PEM_KEY "$($d.FullName)/*" "${SSH_TARGET}:$remotePath/.staging-$($d.Name)/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
|
|
Invoke-Ec2Step "swap in $($d.Name)" "rm -rf $remotePath/$($d.Name) && mv $remotePath/.staging-$($d.Name) $remotePath/$($d.Name)"
|
|
}
|
|
|
|
Write-Host "--- [Done] Static site deploy finished ---" -ForegroundColor Green
|
|
}
|
|
|
|
function Invoke-DockerDeploy {
|
|
param([string]$Key, $Proj)
|
|
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (docker compose) ===" -ForegroundColor Cyan
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
|
|
if (-not (Test-Path -LiteralPath (Join-Path $root "docker-compose.yml"))) { throw "Missing $root\docker-compose.yml" }
|
|
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
Invoke-Ec2Step "ensure project dir" "sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
|
|
|
|
$files = @(Get-ChildItem -LiteralPath $root -File -Force | Where-Object { $_.Name -ne 'nul' })
|
|
foreach ($f in $files) {
|
|
Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan
|
|
scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
# Config subdirectories ship too, exactly as the edge kind does. Uploading
|
|
# top-level files ONLY was silently wrong for any stack that keeps config in
|
|
# a directory: the compose file arrives, the containers restart, and the
|
|
# config they read is whatever was already on the box. That is worse than a
|
|
# failed deploy, because it reports success - a provisioning directory read
|
|
# at container start (alert rules, datasources, mounted *.php) would never
|
|
# reflect the change you just deployed.
|
|
#
|
|
# Skipped: $JunkDirNames (.git, __pycache__, .pytest_cache, ...) plus
|
|
# anything the project lists in deploy.skipDirs. That list is how a stack
|
|
# protects SERVER-SIDE STATE that happens to share the tree - a data/ holding
|
|
# mailboxes or a time-series database must never be overwritten by whatever
|
|
# the local checkout has (usually nothing, which is the dangerous case).
|
|
# .github is CI config - it belongs in the repo and never on a deploy
|
|
# target. It is not in $JunkDirNames because backups DO want it.
|
|
$skipDirs = @($script:JunkDirNames) + @('.github')
|
|
if ($Proj.deploy -and $Proj.deploy.skipDirs) { $skipDirs += @($Proj.deploy.skipDirs) }
|
|
$dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
|
|
foreach ($d in $dirs) {
|
|
Write-Host " >> uploading $($d.Name)/ (recursive)" -ForegroundColor DarkCyan
|
|
scp -r @SCP_OPTS -i $PEM_KEY $d.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
Invoke-Ec2Step "docker compose pull" "cd $remotePath && sudo docker compose pull"
|
|
Invoke-Ec2Step "docker compose up -d" "cd $remotePath && sudo docker compose up -d"
|
|
|
|
Invoke-Ec2PostDeployCleanup -Label $Key
|
|
Write-Host "`n--- [Done] $($Proj.label) deploy finished ---" -ForegroundColor Green
|
|
Write-DeployLocation -Proj $Proj
|
|
}
|
|
|
|
# Pseudo-project "ztokens": not a zconfig entry, no compose stack. Runs
|
|
# `ztokens -Publish` from the sibling ztokens repo so the public zscripts page
|
|
# has current data. A failure here warns rather than aborting the rest of the
|
|
# deploy list - it's a nice-to-have refresh, not a deploy step.
|
|
function Invoke-ZTokensPublish {
|
|
Write-Host "`n=== ztokens: refreshing live-usage stats ===" -ForegroundColor Cyan
|
|
$ztokensScript = Join-Path (Split-Path -Parent $PSScriptRoot) "ztokens\ztokens.ps1"
|
|
if (-not (Test-Path -LiteralPath $ztokensScript)) {
|
|
Write-Host " ztokens.ps1 not found at $ztokensScript - skipping." -ForegroundColor Yellow
|
|
return
|
|
}
|
|
# Post-condition, not decoration. This step ran clean for five days while
|
|
# publishing nothing: ztokens.ps1 had no -Publish switch, and as a simple
|
|
# script PowerShell swallowed the unknown parameter into $args rather than
|
|
# failing. The catch below never fired because nothing threw. So the check
|
|
# is not "did it throw" but "did the file actually move".
|
|
$statsFile = Join-Path (Split-Path -Parent $PSScriptRoot) "www\public\token-stats.json"
|
|
$before = if (Test-Path -LiteralPath $statsFile) { (Get-Item -LiteralPath $statsFile).LastWriteTimeUtc } else { [datetime]::MinValue }
|
|
try {
|
|
& $ztokensScript -Publish
|
|
} catch {
|
|
Write-Host " ztokens -Publish failed: $($_.Exception.Message)" -ForegroundColor Yellow
|
|
return
|
|
}
|
|
$after = if (Test-Path -LiteralPath $statsFile) { (Get-Item -LiteralPath $statsFile).LastWriteTimeUtc } else { [datetime]::MinValue }
|
|
if ($after -le $before) {
|
|
Write-Host " WARNING: token-stats.json was not rewritten - the site will ship the old numbers." -ForegroundColor Yellow
|
|
if ($before -eq [datetime]::MinValue) {
|
|
Write-Host " $statsFile does not exist." -ForegroundColor DarkGray
|
|
} else {
|
|
Write-Host (" Still dated {0:yyyy-MM-dd HH:mm} local." -f $before.ToLocalTime()) -ForegroundColor DarkGray
|
|
}
|
|
Write-Host " Run 'ztokens -Publish' by hand to see why." -ForegroundColor DarkGray
|
|
}
|
|
}
|
|
|
|
# ── Dispatch ─────────────────────────────────────────────────────────────────
|
|
|
|
foreach ($key in $Projects) {
|
|
# 'ztokens' matches the tool it runs (ztokens.cmd / ztokens.ps1). The old
|
|
# singular 'ztoken' still works so existing habits and any script that
|
|
# already calls it keep running.
|
|
if ($key -in @('ztokens', 'ztoken')) { Invoke-ZTokensPublish; continue }
|
|
$proj = Get-ZProject -Key $key
|
|
Invoke-DeployGitPull -Proj $proj # no-op unless deploy.gitPull is set
|
|
switch ([string]$proj.kind) {
|
|
"python" { Invoke-PythonDeploy -Key $key -Proj $proj -ChangeNote $Note }
|
|
"vite" { Invoke-ViteDeploy -Key $key -Proj $proj -ChangeNote $Note }
|
|
"nextjs" { Invoke-NextDeploy -Key $key -Proj $proj -ChangeNote $Note }
|
|
"edge" { Invoke-EdgeDeploy -Key $key -Proj $proj }
|
|
"docker" { Invoke-DockerDeploy -Key $key -Proj $proj }
|
|
"static" { Invoke-StaticDeploy -Key $key -Proj $proj }
|
|
default { throw "No deploy handler for kind '$($proj.kind)' (project '$key'). Add an Invoke-<Kind>Deploy function in zdeploy.ps1." }
|
|
}
|
|
}
|
|
# The timestamp goes through Stop-ZTracking as the FinalNote so it lands after
|
|
# the tracking footer and before the trailing blank lines - the last thing on
|
|
# screen, which is the point: scroll to the bottom and you can see how long ago
|
|
# this deployed. Only reached on success; a failed deploy throws out of the loop
|
|
# above, so this never claims a deploy that didn't happen.
|
|
Stop-ZTracking -FinalNote ("Last deployed at {0}" -f (Get-Date -Format "MM/dd/yyyy hh:mm:ss tt"))
|