mirror of
https://github.com/kellymichels/zscripts-token-savers
synced 2026-10-07 07:18:18 +00:00
b826490019
It was gated behind a single probe that ran immediately after the container was restarted, so the probe hit a port that was not listening yet, the flag stayed false for the whole run, and the check fell back to the public proxy. That answers from whichever vhost matches the Host header, so a deploy that had worked was reported as failing -- against another service's version entirely. Silent and wrong, which is the combination worth removing. viaProxy/upstream is configured explicitly to read a version, so a failure means "not up yet", and the retry loop already handles that: the JSON parse throws on an empty read, the catch waits and tries again. Trusting the config removes the race. The port mechanism keeps its probe -- verify.port is not necessarily a version endpoint, so there the probe answers a real question. CHECKSUMS.txt regenerated; CHANGELOG.md and its plain-text twin carry the entry. Pester: 231 passed, 0 failed.
941 lines
52 KiB
PowerShell
941 lines
52 KiB
PowerShell
# Evomedia.net Token Savers — https://github.com/evomedia-net/evo.zscripts
|
|
# Created by Kelly Michels · dev@evomedia.net
|
|
# Licensed under the MIT License. See LICENSE.
|
|
# Version: v1.0.0.0.20
|
|
|
|
# zdeploy.ps1 — deploy any project defined in zconfig.json to the server.
|
|
# Each project runs its own docker compose stack; the handler is picked by the
|
|
# project's "kind": python | vite | nextjs | edge | docker | static.
|
|
#
|
|
# Usage:
|
|
# zdeploy <project> [<project> ...] [-Note "message"]
|
|
# zdeploy all # ztokens first, then every project (edge kinds next), stop at first failure
|
|
# zdeploy ztokens # refresh the live-usage stats (see below)
|
|
#
|
|
# Examples:
|
|
# zdeploy viteapp
|
|
# zdeploy pyapp -Note "fix billing banner"
|
|
# zdeploy all -Note "weekly release"
|
|
# zdeploy ztokens evo # refresh token-stats.json, then ship the site with it
|
|
#
|
|
# "ztokens" is an OPTIONAL pseudo-project, not a zconfig entry: it runs
|
|
# `ztokens -Publish` from a sibling ztokens checkout, if you have one, to
|
|
# refresh a token-stats.json a site can chart. With no such checkout the step
|
|
# prints a skip and the rest of the run is unaffected. `all` runs it first
|
|
# automatically; called standalone, list it before a site project (as above) so
|
|
# that project's deploy zip picks up the freshly written file.
|
|
#
|
|
# Flow (python/vite/nextjs): zip source -> free server disk space -> scp up ->
|
|
# unzip into remote.path (preserving server-side .env* files and anything in
|
|
# deploy.preserve) -> docker compose build + up -> verify the live site reports
|
|
# the new build version. Zips are always deleted.
|
|
#
|
|
# Compose service-name conventions (override with remote.appService):
|
|
# python kind: app service "app", db service "db"
|
|
# nextjs kind: app service "web", db service "db"
|
|
#
|
|
# Verification (python kind, when there is no scripts/build_version_tool.py):
|
|
# Projects with a "verify" block are checked ON the server via
|
|
# localhost:<port><path> — the only accurate way for stacks that are not
|
|
# published through the edge proxy. Projects with only a "domain" fall back
|
|
# to a Host-header request. Projects with neither are reported as NOT
|
|
# verified rather than passing on the proxy's default vhost.
|
|
#
|
|
param(
|
|
[Parameter(Position = 0, ValueFromRemainingArguments = $true)]
|
|
[string[]]$Projects = @(),
|
|
[string]$Note = "Build deployed"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
. (Join-Path $PSScriptRoot "ZHelpers.ps1")
|
|
Start-ZTracking
|
|
|
|
$cfg = Get-ZConfig
|
|
$EC2_IP = $cfg.ec2.ip
|
|
$PEM_KEY = $cfg.ec2.pemKey
|
|
$STACK_ROOT = $cfg.ec2.stackRoot
|
|
$SSH_TARGET = Get-Ec2Target
|
|
$SSH_OPTS = Get-Ec2SshOpts # see ZHelpers.ps1 - these are what stop a deploy hanging
|
|
$SCP_OPTS = Get-Ec2ScpOpts # same, minus -n: scp rejects it with a usage error
|
|
$RemoteHome = Get-Ec2Home
|
|
$Ec2User = $cfg.ec2.user
|
|
|
|
$TempRoot = $cfg.paths.temp
|
|
if (-not (Test-Path -LiteralPath $TempRoot)) {
|
|
New-Item -ItemType Directory -Path $TempRoot -Force | Out-Null
|
|
}
|
|
|
|
if ($Projects.Count -eq 0) {
|
|
$keys = (Get-ZProjectKeys) -join ', '
|
|
Write-Host ""
|
|
Write-Host "Usage: zdeploy <project> [<project> ...] | all | ztokens [-Note `"message`"]" -ForegroundColor Yellow
|
|
Write-Host " Projects in zconfig.json: $keys" -ForegroundColor Gray
|
|
Write-Host " 'all' deploys everything (edge kinds first) and stops at the first failure." -ForegroundColor Gray
|
|
Write-Host " 'ztokens' refreshes live-usage stats, if a sibling ztokens checkout exists." -ForegroundColor Gray
|
|
Stop-ZTracking; exit 1
|
|
}
|
|
|
|
# Tolerate switch-style args (zdeploy -myproject) from muscle memory.
|
|
$Projects = @($Projects | ForEach-Object { $_.TrimStart('-') })
|
|
|
|
if ($Projects -contains 'all') {
|
|
# 'ztokens' first so any site project deployed below picks up fresh stats.
|
|
$Projects = @('ztokens') + @(Get-ZProjectKeys)
|
|
}
|
|
|
|
# Edge kinds first, however the list was produced. This is a correctness
|
|
# property, not a convenience of 'all': the proxy has to route before the apps
|
|
# behind it ship, or there is a window where a new app is live behind stale
|
|
# routing. `zdeploy evo edge` reads as "these two, edge included" and used to
|
|
# do the risky order, because this sort only ran for 'all'.
|
|
# Order within each group is preserved, so an intentional sequence still holds
|
|
# — notably `zdeploy ztokens evo`, where ztokens must still precede evo.
|
|
$requested = @($Projects)
|
|
$edgeKeys = @($Projects | Where-Object { $cfg.projects.$_.kind -eq 'edge' })
|
|
$restKeys = @($Projects | Where-Object { $cfg.projects.$_.kind -ne 'edge' })
|
|
$Projects = @($edgeKeys + $restKeys)
|
|
|
|
if ($Projects.Count -gt 1) {
|
|
# Say so when the order changed, so the reordering is never silent.
|
|
$note = if (($requested -join ',') -ne ($Projects -join ',')) { " (edge first)" } else { "" }
|
|
Write-Host "Deploying: $($Projects -join ', ')$note" -ForegroundColor Cyan
|
|
}
|
|
|
|
function Get-DeployZipName {
|
|
param([string]$Key, $Proj)
|
|
if ($Proj.deploy -and $Proj.deploy.zipName) { return $Proj.deploy.zipName }
|
|
return "${Key}Deploy.zip"
|
|
}
|
|
|
|
# Pre-upload cleanup: remove stale deploy zips, prune docker, truncate big logs,
|
|
# fail if under 1.5 GB free.
|
|
function Invoke-Ec2PreflightCleanup {
|
|
param([string[]]$ExtraZipsToRemove = @())
|
|
Write-Host "`n--- [Preflight] Freeing disk space on the server ---" -ForegroundColor Cyan
|
|
$rmZipsClause = if ($ExtraZipsToRemove.Count -gt 0) { "rm -f $($ExtraZipsToRemove -join ' ')" } else { "true" }
|
|
$preflightCmd = @(
|
|
"echo '--- df / before cleanup ---'",
|
|
"df -h /",
|
|
"echo '--- removing stale deploy artifacts ---'",
|
|
$rmZipsClause,
|
|
"echo '--- pruning docker build cache + dangling images + stopped containers ---'",
|
|
"sudo docker container prune -f >/dev/null 2>&1 || true",
|
|
"sudo docker builder prune -f >/dev/null 2>&1 || true",
|
|
"sudo docker image prune -af >/dev/null 2>&1 || true",
|
|
"echo '--- truncating large container logs ---'",
|
|
"sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true",
|
|
"echo '--- df / after cleanup ---'",
|
|
"df -h /",
|
|
"avail_mb=`$(df --output=avail -BM / | tail -n 1 | tr -dc 0-9)",
|
|
"[ -z `"`$avail_mb`" ] && avail_mb=0",
|
|
"echo available_mb=`$avail_mb",
|
|
"if [ `"`$avail_mb`" -lt 1500 ]; then echo 'ERROR: less than 1.5 GB free on /. Grow the root volume or run: sudo docker system prune -af' >&2; exit 11; fi"
|
|
) -join '; '
|
|
ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $preflightCmd
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "Server pre-flight cleanup failed (exit $LASTEXITCODE). Root volume too full (need ~1.5 GB free, ideally 3+)."
|
|
}
|
|
}
|
|
|
|
# Post-deploy cleanup: prune build cache and dangling images created during this deploy.
|
|
# Containers/volumes still in use by the running stack are NOT touched.
|
|
function Invoke-Ec2PostDeployCleanup {
|
|
param([string]$Label = "post-deploy")
|
|
Write-Host "`n--- [Post-deploy] Reclaiming disk space ($Label) ---" -ForegroundColor Cyan
|
|
$cmd = @(
|
|
"sudo docker container prune -f >/dev/null 2>&1 || true",
|
|
"sudo docker builder prune -f >/dev/null 2>&1 || true",
|
|
"sudo docker image prune -af >/dev/null 2>&1 || true",
|
|
"sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true",
|
|
"df -h /"
|
|
) -join '; '
|
|
ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $cmd
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-Host " Post-deploy cleanup returned non-zero exit ($LASTEXITCODE); continuing." -ForegroundColor DarkYellow
|
|
}
|
|
}
|
|
|
|
function Send-DeployZip {
|
|
param([string]$LocalZip, [string]$ZipName)
|
|
scp @SCP_OPTS -i $PEM_KEY $LocalZip "${SSH_TARGET}:$RemoteHome/"
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "SCP upload failed (exit $LASTEXITCODE). Read scp's own output above: a usage block means bad arguments, not disk. If it is genuinely full, clear the stale archive: rm -f $RemoteHome/$ZipName"
|
|
}
|
|
}
|
|
|
|
function Invoke-RemoteUnzip {
|
|
param([string]$ZipName, [string]$DestPath)
|
|
$bash = 'test -f {2}/{0} || {{ echo "missing {2}/{0}"; exit 2; }}; unzip -t {2}/{0} || exit 3; unzip -o {2}/{0} -d {1}; uc=$?; if [ $uc -gt 1 ]; then exit $uc; fi; exit 0' -f $ZipName, $DestPath, $RemoteHome
|
|
Invoke-Ec2Step "unzip $ZipName" $bash
|
|
}
|
|
|
|
# ── Operator-file preservation (issue #2) ────────────────────────────────────
|
|
# Deploys replace the project directory wholesale, which used to destroy every
|
|
# operator-managed file except ./.env. These helpers preserve all .env* files
|
|
# at the project root PLUS any paths listed in deploy.preserve (files or
|
|
# directories), by tarring them to the home dir before the wipe and extracting
|
|
# them back after the unzip. Server-side copies win over anything shipped in
|
|
# the zip — the same semantics ./.env always had.
|
|
function Save-OperatorFiles {
|
|
param([string]$Key, $Proj, [string]$RemotePath)
|
|
$paths = @('.env*')
|
|
if ($Proj.deploy -and $Proj.deploy.preserve) { $paths += @($Proj.deploy.preserve) }
|
|
$spec = $paths -join ' '
|
|
$tarball = "$RemoteHome/preserve_${Key}.tgz"
|
|
# NOTE: no embedded quotes or $( ) here - PowerShell 5.1 strips embedded
|
|
# double quotes when passing args to ssh.exe, silently corrupting the
|
|
# remote command. Globs expand remotely; tar archives whatever exists
|
|
# and its nonzero exit for missing paths is deliberately swallowed.
|
|
Invoke-Ec2Step "preserve operator files ($spec)" "rm -f $tarball; cd $RemotePath && tar -czf $tarball $spec 2>/dev/null; true"
|
|
}
|
|
|
|
function Restore-OperatorFiles {
|
|
param([string]$Key, [string]$RemotePath)
|
|
$tarball = "$RemoteHome/preserve_${Key}.tgz"
|
|
Invoke-Ec2Step "restore operator files" "test -f $tarball && tar -xzf $tarball -C $RemotePath; rm -f $tarball; true"
|
|
}
|
|
|
|
# ── Deploy verification (build-version match, not just HTTP 200 — a 200 can be
|
|
# a stale cached build; the version match proves the new build is live) ─────
|
|
|
|
function Wait-VerifyStaticBuild {
|
|
param([string]$Key, $Proj, [object]$PreZipBuildState)
|
|
if (-not $PreZipBuildState) {
|
|
Write-Host "`n--- [$Key version] SKIPPED (no local build-version.json - see 'Enabling deploy verification' in README) ---" -ForegroundColor DarkYellow
|
|
return
|
|
}
|
|
# Expect the COMMITTED stamp, not +1: builds no longer self-bump (a
|
|
# prebuild hook incremented inside the image, so the served version
|
|
# matched no commit and every build dirtied the tree). The counter now
|
|
# advances deliberately — one version bump per release — so "is the
|
|
# build I just packed live?" means an exact match.
|
|
$expectedLabel = Get-LabelFromBuildJsonObj $PreZipBuildState
|
|
Write-Host "`n--- [$Key] Live build verification (expect $expectedLabel) ---" -ForegroundColor Cyan
|
|
$containerName = $Proj.remote.containerName
|
|
$deadline = (Get-Date).AddSeconds(45)
|
|
while ((Get-Date) -lt $deadline) {
|
|
try {
|
|
$r = $null
|
|
if ($containerName) {
|
|
# build-version.json may be blocked from external requests by the edge
|
|
# proxy; read it inside the running container instead.
|
|
$raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET `
|
|
"sudo docker exec $containerName cat /usr/share/nginx/html/build-version.json 2>/dev/null"
|
|
if ($raw) { $r = $raw | ConvertFrom-Json -ErrorAction Stop }
|
|
} else {
|
|
$headers = @{}
|
|
if ($Proj.domain) { $headers['Host'] = $Proj.domain }
|
|
$r = Invoke-RestMethod -Uri "http://$EC2_IP/build-version.json" -Headers $headers -TimeoutSec 10 -ErrorAction Stop
|
|
}
|
|
if ($r) {
|
|
$remoteLabel = Get-LabelFromBuildJsonObj $r
|
|
if ($remoteLabel -eq $expectedLabel) {
|
|
Write-Host " PASS - live build $remoteLabel matches expected." -ForegroundColor Green
|
|
return
|
|
}
|
|
Write-Host " Live build is $remoteLabel, expected $expectedLabel - waiting..." -ForegroundColor DarkYellow
|
|
}
|
|
} catch {
|
|
Write-Host " Container not ready yet - waiting..." -ForegroundColor DarkGray
|
|
}
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
Write-Host " WARNING: live build did not match $expectedLabel within 45s (upload or Docker build may have failed, or a stale build is cached)." -ForegroundColor Yellow
|
|
}
|
|
|
|
function Wait-VerifyApiBuild {
|
|
param([string]$Key, $Proj, [string]$ExpectedLabel, [int]$TimeoutSec = 60)
|
|
Write-Host "`n--- [$Key] Live build verification (expect $ExpectedLabel) ---" -ForegroundColor Cyan
|
|
$headers = @{}
|
|
# deploy.verifyHost overrides domain for verification only. The Host header
|
|
# decides which edge vhost answers, and a project's public host can be
|
|
# deliberately unroutable while the app is perfectly healthy - that is why
|
|
# the override exists. Reach for it when a domain is being retired ahead of
|
|
# its replacement: the old host may be returning 410 while the new one has
|
|
# no DNS yet, so neither answers even though the app is fine.
|
|
$verifyHost = if ($Proj.deploy -and $Proj.deploy.verifyHost) { $Proj.deploy.verifyHost } else { $Proj.domain }
|
|
if ($verifyHost) { $headers['Host'] = $verifyHost }
|
|
# Preferred when the project configures it: read the version from a
|
|
# container ON the shared docker network rather than through the public
|
|
# proxy. A service that publishes no port cannot be curled from the host
|
|
# at all, and the proxy answers from whichever vhost matches the Host
|
|
# header - so a container with no public route gets another site's
|
|
# version back. See Get-ServerSideVersionCommand.
|
|
$execCmd = Get-ServerSideVersionCommand -Proj $Proj
|
|
# No pre-probe. viaProxy/upstream is configured EXPLICITLY to read a
|
|
# version, so a failure here means "not up yet" - which is what the retry
|
|
# loop below exists for.
|
|
#
|
|
# Gating on one probe made the app's own restart a race, and losing it
|
|
# was silent AND wrong: this runs right after the container is restarted,
|
|
# so the probe hit a port that was not listening, $useExec stayed false
|
|
# for the whole run, and the check fell back to the proxy - which answers
|
|
# from whichever vhost matches the Host header, i.e. another service's
|
|
# version. The deploy had worked; only the verification was lying.
|
|
$useExec = [bool]$execCmd
|
|
if ($useExec) {
|
|
Write-Host " Asking on server: $($Proj.verify.upstream) (via $($Proj.verify.viaProxy))" -ForegroundColor DarkGray
|
|
}
|
|
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
while ((Get-Date) -lt $deadline) {
|
|
try {
|
|
if ($useExec) {
|
|
$raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $execCmd
|
|
$live = Get-LabelFromVersionJson ($raw | Out-String)
|
|
} else {
|
|
$r = Invoke-RestMethod -Uri "http://$EC2_IP/api/build-version" -Headers $headers -TimeoutSec 10 -ErrorAction Stop
|
|
$live = if ($r -and $r.build_version) { [string]$r.build_version } else { $null }
|
|
}
|
|
if ($live) {
|
|
if ($live -eq $ExpectedLabel) {
|
|
Write-Host " PASS - live build $live matches expected." -ForegroundColor Green
|
|
return $true
|
|
}
|
|
Write-Host " Live build is $live, expected $ExpectedLabel - waiting..." -ForegroundColor DarkYellow
|
|
}
|
|
} catch {
|
|
Write-Host " version endpoint not ready yet - waiting..." -ForegroundColor DarkGray
|
|
}
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
Write-Host " WARNING: live build did not match $ExpectedLabel within ${TimeoutSec}s (a stale build may be cached)." -ForegroundColor Yellow
|
|
return $false
|
|
}
|
|
|
|
# Verify a deploy by calling the app ON the server (localhost:<port>). Works
|
|
# for stacks that are not published through the edge proxy or whose host port
|
|
# is closed to the internet — hitting http://<ec2-ip>/ for those just answers
|
|
# from whatever vhost the proxy serves by default, which is a false PASS.
|
|
#
|
|
# Configure per project in zconfig.json:
|
|
# "verify": { "port": 8005, "path": "/health", "expect": "\"status\":\"ok\"" }
|
|
# port is required; path defaults to "/", expect is an optional substring.
|
|
function Test-DeployHealth {
|
|
param([string]$Key, $Proj, [int]$TimeoutSec = 60)
|
|
|
|
$port = [int]$Proj.verify.port
|
|
$path = if ($Proj.verify.path) { [string]$Proj.verify.path } else { "/" }
|
|
$expect = [string]$Proj.verify.expect
|
|
|
|
Write-Host "`n--- [$Key] Health check (on server: localhost:$port$path) ---" -ForegroundColor Cyan
|
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
|
while ((Get-Date) -lt $deadline) {
|
|
# -L: an app whose "/" redirects (e.g. Next.js "/" -> "/login") answers a
|
|
# 307 whose body is a few bytes or empty, which reads as "not ready".
|
|
# Follow to the page that actually renders before judging.
|
|
$raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "curl -sL -m 8 http://localhost:$port$path"
|
|
$body = ($raw | Out-String).Trim()
|
|
if ($LASTEXITCODE -eq 0 -and $body) {
|
|
if (-not $expect -or $body.Contains($expect)) {
|
|
# A /health endpoint returns a line of JSON; an app page returns
|
|
# kilobytes of HTML. Match on the whole body, but only print
|
|
# enough to recognise it — the rest is unreadable in a log.
|
|
$shown = if ($body.Length -gt 200) { $body.Substring(0, 200) + "... ($($body.Length) bytes)" } else { $body }
|
|
Write-Host " PASS - $shown" -ForegroundColor Green
|
|
return $true
|
|
}
|
|
Write-Host " Responding but '$expect' not found - waiting..." -ForegroundColor DarkYellow
|
|
} else {
|
|
Write-Host " Not ready yet - waiting..." -ForegroundColor DarkGray
|
|
}
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
Write-Host " WARNING: no healthy response from localhost:$port$path within ${TimeoutSec}s." -ForegroundColor Yellow
|
|
return $false
|
|
}
|
|
|
|
# Where the thing just deployed can be reached. A project published through the
|
|
# edge proxy has a domain; one that is not still has somewhere to point at, and
|
|
# saying nothing is the least useful option — an internal service is exactly the
|
|
# case where "where did that land?" is hardest to answer from memory. Falls back
|
|
# through what the project actually declares, and prints nothing if it declares
|
|
# none of it.
|
|
function Write-DeployLocation {
|
|
param($Proj, [int]$Pad = 0)
|
|
$label = "Site:".PadRight([Math]::Max(5, $Pad))
|
|
if ($Proj.domain) {
|
|
Write-Host "$label https://$($Proj.domain)" -ForegroundColor Yellow
|
|
return
|
|
}
|
|
# No public route: give the server-local endpoint the deploy just verified.
|
|
$port = if ($Proj.verify -and $Proj.verify.port) { [int]$Proj.verify.port }
|
|
elseif ($Proj.ports -and $Proj.ports.prod) { [int]$Proj.ports.prod }
|
|
else { 0 }
|
|
if ($port -le 0) { return }
|
|
$path = if ($Proj.verify -and $Proj.verify.path) { [string]$Proj.verify.path } else { "" }
|
|
Write-Host "$label http://127.0.0.1:$port$path (on the server; no public domain)" -ForegroundColor Yellow
|
|
}
|
|
|
|
# ── Kind handlers ────────────────────────────────────────────────────────────
|
|
|
|
function Invoke-PythonDeploy {
|
|
param([string]$Key, $Proj, [string]$ChangeNote)
|
|
|
|
$DeployStart = Get-Date
|
|
$prevLoc = Get-Location
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
$composeDir = if ($Proj.remote.composeDir) { $Proj.remote.composeDir } else { $remotePath }
|
|
$appSvc = if ($Proj.remote.appService) { $Proj.remote.appService } else { "app" }
|
|
$zipName = Get-DeployZipName -Key $Key -Proj $Proj
|
|
$zipLocal = Join-Path $TempRoot $zipName
|
|
$BuildVersion = $null
|
|
$versionTool = Join-Path $root "scripts\build_version_tool.py"
|
|
$hasVersionTool = Test-Path -LiteralPath $versionTool
|
|
|
|
try {
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
|
|
Set-Location -LiteralPath $root
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (python) ===" -ForegroundColor Cyan
|
|
Write-Host "Local zip: $zipLocal" -ForegroundColor DarkGray
|
|
|
|
Write-Host "`n--- [1] Zipping $($Proj.label) ---" -ForegroundColor Cyan
|
|
Get-ChildItem -LiteralPath $root -Directory -Recurse -Filter "__pycache__" -ErrorAction SilentlyContinue |
|
|
Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
|
|
New-ProjectArchive -SourcePath $root -DestinationZip $zipLocal -TopLevelExclude (Get-ArchiveExcludes -Project $Proj)
|
|
|
|
Invoke-Ec2PreflightCleanup -ExtraZipsToRemove @("$RemoteHome/$zipName")
|
|
|
|
Write-Host "`n--- [2] Uploading zip ---" -ForegroundColor Cyan
|
|
Send-DeployZip -LocalZip $zipLocal -ZipName $zipName
|
|
|
|
Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
|
|
Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
|
|
Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
|
|
Invoke-Ec2Step "replace project directory" "sudo rm -rf $remotePath && sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
|
|
Invoke-RemoteUnzip -ZipName $zipName -DestPath $remotePath
|
|
Restore-OperatorFiles -Key $Key -RemotePath $remotePath
|
|
Invoke-Ec2Step "require compose directory" "test -d $composeDir"
|
|
Invoke-Ec2Step "docker compose build $appSvc" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose build $appSvc"
|
|
Invoke-Ec2Step "docker compose up -d" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose up -d"
|
|
Invoke-Ec2Step "record deploy time; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remotePath/.last_deploy_utc > /dev/null && rm -f $RemoteHome/$zipName"
|
|
|
|
if ($hasVersionTool) {
|
|
Write-Host "`n--- [4] Incrementing build version ---" -ForegroundColor Cyan
|
|
$BumpCmd = "cd $composeDir && sudo docker compose exec -T $appSvc python scripts/build_version_tool.py bump"
|
|
for ($attempt = 1; $attempt -le 5; $attempt++) {
|
|
$output = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $BumpCmd
|
|
if ($LASTEXITCODE -eq 0 -and $output) {
|
|
$BuildVersion = ($output | Select-Object -Last 1).ToString().Trim()
|
|
break
|
|
}
|
|
Write-Host " Attempt $attempt failed, retrying in 3s..." -ForegroundColor DarkYellow
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
if (-not $BuildVersion) { throw "Build version bump failed after 5 attempts" }
|
|
|
|
# The local stamp is deliberately NOT mirrored back. Writing it
|
|
# left build-version.json dirty after every deploy, and committing
|
|
# that hit branch protection ("Changes must be made through a pull
|
|
# request") — so each deploy either tripped the NEXT deploy's
|
|
# clean-tree guard or bypassed the rule. Neither is acceptable as
|
|
# routine behaviour.
|
|
#
|
|
# The repo file now records the STAGE baseline only (it changes on
|
|
# a stage bump, through a normal PR). The live build number lives
|
|
# in the container, is written to .build_version below, and is
|
|
# proven by the /api/build-version check — which is the thing that
|
|
# actually establishes what is deployed.
|
|
ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "echo '$BuildVersion' | sudo tee $remotePath/.build_version > /dev/null"
|
|
$changelogTool = Join-Path $root "scripts\build_changelog_tool.py"
|
|
if (Test-Path -LiteralPath $changelogTool) {
|
|
if ([string]::IsNullOrWhiteSpace($ChangeNote)) { $ChangeNote = "Build deployed" }
|
|
python $changelogTool append --version $BuildVersion --note "$ChangeNote" | Out-Null
|
|
}
|
|
|
|
Write-Host "`n--- [5] Restarting app to pick up new version ---" -ForegroundColor Cyan
|
|
ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "cd $composeDir && sudo COMPOSE_BAKE=false docker compose restart $appSvc"
|
|
if ($LASTEXITCODE -ne 0) { throw "App restart after build bump failed (exit $LASTEXITCODE)" }
|
|
|
|
Wait-VerifyApiBuild -Key $Key -Proj $Proj -ExpectedLabel $BuildVersion -TimeoutSec 30 | Out-Null
|
|
} elseif ($Proj.verify -and $Proj.verify.port) {
|
|
Test-DeployHealth -Key $Key -Proj $Proj -TimeoutSec 60 | Out-Null
|
|
} elseif ($Proj.domain) {
|
|
Write-Host "`n--- [4] Basic reachability check (no build_version_tool - see 'Enabling deploy verification' in README) ---" -ForegroundColor Cyan
|
|
$headers = @{ 'Host' = $Proj.domain }
|
|
$deadline = (Get-Date).AddSeconds(30)
|
|
$up = $false
|
|
while ((Get-Date) -lt $deadline) {
|
|
Start-Sleep -Seconds 3
|
|
try {
|
|
$resp = Invoke-WebRequest -Uri "http://$EC2_IP/" -Headers $headers -UseBasicParsing -TimeoutSec 8 -ErrorAction Stop
|
|
if ($resp.StatusCode -lt 500) { $up = $true; break }
|
|
} catch { Write-Host " App not ready yet - waiting..." -ForegroundColor DarkGray }
|
|
}
|
|
if ($up) { Write-Host " App is responding." -ForegroundColor Green }
|
|
else { Write-Host " WARNING: app did not respond within 30s." -ForegroundColor Yellow }
|
|
} else {
|
|
# No domain to send as a Host header and no "verify" block: a request
|
|
# to http://<ec2-ip>/ would be answered by the proxy's default vhost,
|
|
# so it proves nothing about THIS app. Say so instead of faking a PASS.
|
|
Write-Host "`n--- [4] Deploy finished - NOT verified ---" -ForegroundColor Yellow
|
|
Write-Host " No 'domain' and no 'verify' block in zconfig.json for '$Key'," -ForegroundColor Yellow
|
|
Write-Host " so there is no way to confirm the new build is live." -ForegroundColor Yellow
|
|
Write-Host ' Add to the project: "verify": { "port": <hostPort>, "path": "/health" }' -ForegroundColor Gray
|
|
}
|
|
|
|
Invoke-Ec2PostDeployCleanup -Label $Key
|
|
|
|
$Elapsed = (Get-Date) - $DeployStart
|
|
$ElapsedStr = "{0:mm\:ss}" -f $Elapsed
|
|
Write-Host "`n--- [Done] $($Proj.label) deployed! ---" -ForegroundColor Green
|
|
Write-DeployLocation -Proj $Proj
|
|
if ($BuildVersion) { Write-Host "Build Version: $BuildVersion" -ForegroundColor Magenta }
|
|
Write-Host "Change Note: $ChangeNote" -ForegroundColor Cyan
|
|
Write-Host "Deploy Time: $ElapsedStr ($([math]::Round($Elapsed.TotalSeconds))s)" -ForegroundColor DarkGray
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $zipLocal) {
|
|
try { Remove-Item -LiteralPath $zipLocal -Force -ErrorAction Stop } catch { }
|
|
}
|
|
Set-Location -LiteralPath $prevLoc
|
|
}
|
|
}
|
|
|
|
function Invoke-ViteDeploy {
|
|
param([string]$Key, $Proj, [string]$ChangeNote)
|
|
|
|
$DeployStart = Get-Date
|
|
$prevLoc = Get-Location
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
$zipName = Get-DeployZipName -Key $Key -Proj $Proj
|
|
$zipLocal = Join-Path $TempRoot $zipName
|
|
$preZipBuild = $null
|
|
|
|
try {
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
|
|
Set-Location -LiteralPath $root
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (vite/static) ===" -ForegroundColor Cyan
|
|
Write-Host "Local zip: $zipLocal" -ForegroundColor DarkGray
|
|
|
|
Write-Host "`n--- [1] Zipping site ---" -ForegroundColor Cyan
|
|
$preZipBuild = Read-JsonBuildVersion -FilePath (Join-Path $root "build-version.json")
|
|
if ($preZipBuild) {
|
|
Write-Host " Pre-zip build label: $(Get-LabelFromBuildJsonObj $preZipBuild) (verification expects this exact label)" -ForegroundColor Gray
|
|
}
|
|
New-ProjectArchive -SourcePath $root -DestinationZip $zipLocal -TopLevelExclude (Get-ArchiveExcludes -Project $Proj)
|
|
|
|
Invoke-Ec2PreflightCleanup -ExtraZipsToRemove @("$RemoteHome/$zipName")
|
|
|
|
Write-Host "`n--- [2] Uploading zip ---" -ForegroundColor Cyan
|
|
Send-DeployZip -LocalZip $zipLocal -ZipName $zipName
|
|
|
|
Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
|
|
Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
|
|
Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
|
|
Invoke-Ec2Step "replace project directory" "sudo rm -rf $remotePath && sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
|
|
Invoke-RemoteUnzip -ZipName $zipName -DestPath $remotePath
|
|
Restore-OperatorFiles -Key $Key -RemotePath $remotePath
|
|
Invoke-Ec2Step "require compose file" "test -f $remotePath/docker-compose.yml"
|
|
Invoke-Ec2Step "docker compose build" "cd $remotePath && sudo COMPOSE_BAKE=false docker compose build"
|
|
Invoke-Ec2Step "docker compose up -d" "cd $remotePath && sudo COMPOSE_BAKE=false docker compose up -d"
|
|
|
|
$edgeProj = Get-ZEdgeProject
|
|
if ($edgeProj -and $edgeProj.Config.proxyContainer) {
|
|
Invoke-Ec2Step "reload edge nginx (flush DNS cache for new container IP)" "sudo docker exec $($edgeProj.Config.proxyContainer) nginx -s reload"
|
|
}
|
|
Invoke-Ec2Step "record deploy time; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remotePath/.last_deploy_utc > /dev/null && rm -f $RemoteHome/$zipName"
|
|
|
|
Wait-VerifyStaticBuild -Key $Key -Proj $Proj -PreZipBuildState $preZipBuild
|
|
Invoke-Ec2PostDeployCleanup -Label $Key
|
|
|
|
$Elapsed = (Get-Date) - $DeployStart
|
|
$ElapsedStr = "{0:mm\:ss}" -f $Elapsed
|
|
Write-Host "`n--- [Done] $($Proj.label) deployed! ---" -ForegroundColor Green
|
|
Write-DeployLocation -Proj $Proj
|
|
Write-Host "Change Note: $ChangeNote" -ForegroundColor Cyan
|
|
Write-Host "Deploy Time: $ElapsedStr ($([math]::Round($Elapsed.TotalSeconds))s)" -ForegroundColor DarkGray
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $zipLocal) {
|
|
try { Remove-Item -LiteralPath $zipLocal -Force -ErrorAction Stop } catch { }
|
|
}
|
|
Set-Location -LiteralPath $prevLoc
|
|
}
|
|
}
|
|
|
|
function Invoke-NextDeploy {
|
|
param([string]$Key, $Proj, [string]$ChangeNote)
|
|
|
|
$DeployStart = Get-Date
|
|
$prevLoc = Get-Location
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
# Same resolution as the python handler. Must be computed HERE: PowerShell
|
|
# function scope means the copy in Invoke-PythonDeploy is invisible from
|
|
# this one, and an unset variable interpolates to an empty string — so
|
|
# "cd && docker compose down" quietly runs in the home directory.
|
|
$composeDir = if ($Proj.remote.composeDir) { $Proj.remote.composeDir } else { $remotePath }
|
|
$appSvc = if ($Proj.remote.appService) { $Proj.remote.appService } else { "web" }
|
|
$zipName = Get-DeployZipName -Key $Key -Proj $Proj
|
|
$zipLocal = Join-Path $TempRoot $zipName
|
|
$preZipBuild = $null
|
|
|
|
try {
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
|
|
Set-Location -LiteralPath $root
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (nextjs) ===" -ForegroundColor Cyan
|
|
Write-Host "Local zip: $zipLocal" -ForegroundColor DarkGray
|
|
|
|
# Stamp the build version from git before zipping, if the project says
|
|
# how (deploy.stampCmd in zconfig). The image has no .git - it is in the
|
|
# archive excludes - so the number has to be written on this side of the
|
|
# zip.
|
|
#
|
|
# Written, zipped, then reverted: zdeploy refuses a dirty tree, so a
|
|
# stamp that dirtied it every deploy would block the next one. The
|
|
# committed file stays a fallback for local dev; the number that ships
|
|
# is derived from the commit being deployed.
|
|
$stampCmd = if ($Proj.deploy -and $Proj.deploy.stampCmd) { $Proj.deploy.stampCmd } else { $null }
|
|
$stampFile = if ($Proj.deploy -and $Proj.deploy.stampFile) { $Proj.deploy.stampFile } else { "public/build-version.json" }
|
|
$stampedLabel = $null
|
|
if ($stampCmd) {
|
|
Write-Host "`n--- [0] Stamping build version from git ---" -ForegroundColor Cyan
|
|
$out = & cmd /c $stampCmd 2>&1
|
|
if ($LASTEXITCODE -ne 0) { throw "Build stamp failed: $out" }
|
|
$stampedLabel = ($out | Select-Object -Last 1).ToString().Trim()
|
|
Write-Host " $stampedLabel (derived from the commit, not a counter)" -ForegroundColor Gray
|
|
}
|
|
|
|
$preZipBuild = Read-JsonBuildVersion -FilePath (Join-Path $root "public\build-version.json")
|
|
if ($preZipBuild -and -not $stampedLabel) {
|
|
Write-Host " Pre-zip build label: $(Get-LabelFromBuildJsonObj $preZipBuild)" -ForegroundColor Gray
|
|
}
|
|
|
|
Write-Host "`n--- [1] Zipping project files ---" -ForegroundColor Cyan
|
|
New-ProjectArchive -SourcePath $root -DestinationZip $zipLocal -TopLevelExclude (Get-ArchiveExcludes -Project $Proj)
|
|
|
|
if ($stampCmd) {
|
|
# Tree back to clean now the number is inside the archive.
|
|
git -C $root checkout -- $stampFile 2>&1 | Out-Null
|
|
}
|
|
|
|
Invoke-Ec2PreflightCleanup -ExtraZipsToRemove @("$RemoteHome/$zipName")
|
|
|
|
Write-Host "`n--- [2] Uploading zip ---" -ForegroundColor Cyan
|
|
Send-DeployZip -LocalZip $zipLocal -ZipName $zipName
|
|
|
|
Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
|
|
Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
|
|
Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
|
|
Invoke-Ec2Step "replace project directory" "sudo rm -rf $remotePath && sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
|
|
Invoke-RemoteUnzip -ZipName $zipName -DestPath $remotePath
|
|
Restore-OperatorFiles -Key $Key -RemotePath $remotePath
|
|
|
|
Write-Host "`n--- [4] Docker compose rebuild ---" -ForegroundColor Cyan
|
|
# composeDir, not remotePath: a project whose compose lives in a
|
|
# subdirectory (deploy/, infra/, ...) otherwise runs these against
|
|
# whatever docker-compose.yml happens to sit at the project root. That
|
|
# is usually the LOCAL DEV compose, which ships in the same archive —
|
|
# so the deploy recycled a dev database, found no app service to build,
|
|
# exited 0, and left the previous image serving. Verification passed
|
|
# because the untouched old container still answered. Two deploys in a
|
|
# row silently shipped nothing.
|
|
# Assert locally, not just on the server: `test -d $composeDir` with an
|
|
# empty value becomes bare `test -d`, which is TRUE (one non-empty
|
|
# argument), so the remote guard cannot catch this.
|
|
if ([string]::IsNullOrWhiteSpace($composeDir)) { throw "composeDir resolved empty for '$Key' - compose would run in the wrong directory." }
|
|
Invoke-Ec2Step "require compose file" "test -f $composeDir/docker-compose.yml || test -f $composeDir/docker-compose.yaml"
|
|
# BUILD BEFORE DOWN. This used to run `down` first, which took the site
|
|
# offline for the whole build - minutes for a Next.js app - and left it
|
|
# offline if the build failed. That is not hypothetical: one deploy
|
|
# stopped the stack, the build did not finish, and the site served 502
|
|
# for 19 hours with no container running at all. The
|
|
# old image keeps serving while the new one builds, so a failed build is
|
|
# now harmless and the outage is the seconds between down and up.
|
|
#
|
|
# Named service, like the python handler: a compose file that does not
|
|
# define it fails here instead of succeeding with nothing to do.
|
|
Invoke-Ec2Step "docker compose build $appSvc" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose build $appSvc"
|
|
Invoke-Ec2Step "docker compose down" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose down"
|
|
Invoke-Ec2Step "docker compose up -d" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose up -d"
|
|
|
|
if ($Proj.db -and $Proj.db.user -and $Proj.db.name) {
|
|
$waitDb = "cd $composeDir && for i in `$(seq 1 30); do sudo docker compose exec -T db pg_isready -U $($Proj.db.user) -d $($Proj.db.name) >/dev/null 2>&1 && break; sleep 2; done"
|
|
Invoke-Ec2Step "wait for postgres ready" $waitDb
|
|
}
|
|
if ($Proj.migrations -eq "prisma") {
|
|
Invoke-Ec2Step "apply prisma migrations" "cd $composeDir && sudo docker compose exec -T $appSvc npx prisma migrate deploy"
|
|
}
|
|
Invoke-Ec2Step "record deploy timestamp; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remotePath/.last_deploy_utc > /dev/null && rm -f $RemoteHome/$zipName"
|
|
|
|
Write-Host "`n--- [5] Verifying deployment ---" -ForegroundColor Cyan
|
|
# Same precedence as the python handler. Do NOT probe http://<ec2-ip>:<prod-port>/
|
|
# here: a compose stack behind the edge proxy usually publishes to
|
|
# 127.0.0.1 only, so that probe can never answer and the old "is the port
|
|
# open in the security group?" warning sent you chasing a firewall rule
|
|
# for an app that was already up.
|
|
if ($Proj.verify -and $Proj.verify.port) {
|
|
Test-DeployHealth -Key $Key -Proj $Proj -TimeoutSec 60 | Out-Null
|
|
} elseif ($Proj.domain) {
|
|
$headers = @{ 'Host' = $Proj.domain }
|
|
$deadline = (Get-Date).AddSeconds(60)
|
|
$verified = $false
|
|
while ((Get-Date) -lt $deadline) {
|
|
Start-Sleep -Seconds 4
|
|
try {
|
|
$resp = Invoke-WebRequest -Uri "http://$EC2_IP/" -Headers $headers -TimeoutSec 8 -ErrorAction Stop -UseBasicParsing
|
|
if ($resp.StatusCode -lt 500) {
|
|
Write-Host " PASS - app is responding at https://$($Proj.domain)/" -ForegroundColor Green
|
|
$verified = $true
|
|
break
|
|
}
|
|
} catch {
|
|
Write-Host " App not ready yet - waiting..." -ForegroundColor DarkGray
|
|
}
|
|
}
|
|
if (-not $verified) {
|
|
Write-Host " WARNING: no response for https://$($Proj.domain)/ within 60s." -ForegroundColor Yellow
|
|
}
|
|
} else {
|
|
Write-Host " Deploy finished - NOT verified." -ForegroundColor Yellow
|
|
Write-Host " No 'domain' and no 'verify' block in zconfig.json for '$Key'." -ForegroundColor Yellow
|
|
}
|
|
if ($stampedLabel -or $preZipBuild) {
|
|
# The label that actually went into the archive: the derived one
|
|
# when the project stamps, otherwise the committed stamp.
|
|
$expectedLabel = if ($stampedLabel) { $stampedLabel } else { Get-LabelFromBuildJsonObj $preZipBuild }
|
|
Wait-VerifyApiBuild -Key $Key -Proj $Proj -ExpectedLabel $expectedLabel -TimeoutSec 60 | Out-Null
|
|
} else {
|
|
Write-Host " (No public/build-version.json - version verification skipped. See 'Enabling deploy verification' in README.)" -ForegroundColor DarkYellow
|
|
}
|
|
|
|
Invoke-Ec2PostDeployCleanup -Label $Key
|
|
|
|
$Elapsed = (Get-Date) - $DeployStart
|
|
$ElapsedStr = "{0:mm\:ss}" -f $Elapsed
|
|
Write-Host "`n--- [Done] $($Proj.label) deployed! ---" -ForegroundColor Green
|
|
Write-DeployLocation -Proj $Proj -Pad 12
|
|
Write-Host "Change Note: $ChangeNote" -ForegroundColor Cyan
|
|
Write-Host "Deploy Time: $ElapsedStr ($([math]::Round($Elapsed.TotalSeconds))s)" -ForegroundColor DarkGray
|
|
}
|
|
finally {
|
|
if (Test-Path -LiteralPath $zipLocal) {
|
|
try { Remove-Item -LiteralPath $zipLocal -Force -ErrorAction Stop } catch { }
|
|
}
|
|
Set-Location -LiteralPath $prevLoc
|
|
}
|
|
}
|
|
|
|
function Invoke-EdgeDeploy {
|
|
param([string]$Key, $Proj)
|
|
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
$pc = $Proj.proxyContainer
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (edge nginx ingress) ===" -ForegroundColor Cyan
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Edge root not found: $root" }
|
|
foreach ($required in @('docker-compose.yml', 'nginx.conf')) {
|
|
if (-not (Test-Path -LiteralPath (Join-Path $root $required))) { throw "Missing $root\$required" }
|
|
}
|
|
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
# -R: docker creates mount-point subdirs (vendor/, fonts/) root-owned when
|
|
# they are missing at compose up; a non-recursive chown leaves those
|
|
# unwritable and every scp into them fails.
|
|
Invoke-Ec2Step "ensure edge dir" "sudo mkdir -p $remotePath && sudo chown -R ${Ec2User}:${Ec2User} $remotePath"
|
|
|
|
# Ship every top-level file in the edge folder — nginx.conf, compose, css,
|
|
# htpasswd, whatever the proxy serves.
|
|
$files = @(Get-ChildItem -LiteralPath $root -File | Where-Object { $_.Name -ne 'nul' })
|
|
foreach ($f in $files) {
|
|
Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan
|
|
scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
# Content subdirectories the proxy serves (fonts/, vendor/, ...) ship too —
|
|
# only server-side state stays put. Skipping them is how self-hosted assets
|
|
# silently never reach prod: docker creates empty mount-point dirs and nginx
|
|
# serves 404s from them, so fonts fall back and vendored JS never loads.
|
|
$skipDirs = @('nginx-logs', '.git')
|
|
$dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
|
|
foreach ($d in $dirs) {
|
|
Write-Host " >> uploading $($d.Name)/ (recursive)" -ForegroundColor DarkCyan
|
|
scp -r @SCP_OPTS -i $PEM_KEY $d.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
$certMount = if ($Proj.certsSource) { "-v $($Proj.certsSource):/etc/letsencrypt/:ro " } else { "" }
|
|
Invoke-Ec2Step "validate new nginx.conf" "sudo docker run --rm -v $remotePath/nginx.conf:/etc/nginx/nginx.conf:ro ${certMount}nginx:1.27-alpine nginx -t -c /etc/nginx/nginx.conf"
|
|
|
|
# A container from an older compose project may still hold the proxy name;
|
|
# docker refuses a second create with the same name, so remove it first.
|
|
$rmStale = if ($pc) { "; sudo docker rm -f $pc 2>/dev/null || true" } else { "" }
|
|
Invoke-Ec2Step "edge: compose down + remove stale proxy" "cd $remotePath && sudo docker compose down 2>/dev/null || true$rmStale"
|
|
Invoke-Ec2Step "edge compose up -d" "cd $remotePath && sudo docker compose up -d"
|
|
if ($pc) {
|
|
Invoke-Ec2Step "edge nginx reload" "sudo docker exec $pc nginx -s reload || true"
|
|
}
|
|
Invoke-Ec2Step "fix nginx-logs permissions (if present)" "if [ -d $remotePath/nginx-logs ]; then sudo chmod 777 $remotePath/nginx-logs; sudo chmod 666 $remotePath/nginx-logs/*.log 2>/dev/null || true; fi"
|
|
Write-Host "--- [Done] Edge proxy deploy finished ---" -ForegroundColor Green
|
|
}
|
|
|
|
function Invoke-StaticDeploy {
|
|
param([string]$Key, $Proj)
|
|
|
|
# Plain static sites - no build, no container of their own. A shared web
|
|
# container serves them straight off disk, so shipping the files IS the
|
|
# deploy: there is nothing to restart afterwards.
|
|
$root = Join-Path $Proj.localRoot $Proj.siteDir
|
|
$remotePath = $Proj.remote.path
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (static files) ===" -ForegroundColor Cyan
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Static site root not found: $root" }
|
|
if (-not (Test-Path -LiteralPath (Join-Path $root 'index.html'))) { throw "Missing $root\index.html" }
|
|
|
|
Invoke-Ec2Step "ensure site dir" "sudo mkdir -p $remotePath && sudo chown -R ${Ec2User}:${Ec2User} $remotePath"
|
|
|
|
$files = @(Get-ChildItem -LiteralPath $root -File)
|
|
foreach ($f in $files) {
|
|
Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan
|
|
scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
# A large media file uploaded in place is served half-written to anyone who
|
|
# requests it mid-copy. Ship each directory to a sibling, then swap it in -
|
|
# the swap is a rename, so the switch is atomic and visitors never see a
|
|
# partial file.
|
|
$skipDirs = @($script:JunkDirNames) + @('.github')
|
|
if ($Proj.deploy -and $Proj.deploy.skipDirs) { $skipDirs += @($Proj.deploy.skipDirs) }
|
|
$dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
|
|
foreach ($d in $dirs) {
|
|
Write-Host " >> uploading $($d.Name)/ (recursive, staged)" -ForegroundColor DarkCyan
|
|
Invoke-Ec2Step "stage $($d.Name)" "rm -rf $remotePath/.staging-$($d.Name) && mkdir -p $remotePath/.staging-$($d.Name)"
|
|
scp -r @SCP_OPTS -i $PEM_KEY "$($d.FullName)/*" "${SSH_TARGET}:$remotePath/.staging-$($d.Name)/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
|
|
Invoke-Ec2Step "swap in $($d.Name)" "rm -rf $remotePath/$($d.Name) && mv $remotePath/.staging-$($d.Name) $remotePath/$($d.Name)"
|
|
}
|
|
|
|
Write-Host "--- [Done] Static site deploy finished ---" -ForegroundColor Green
|
|
}
|
|
|
|
function Invoke-DockerDeploy {
|
|
param([string]$Key, $Proj)
|
|
|
|
$root = $Proj.localRoot
|
|
$remotePath = $Proj.remote.path
|
|
|
|
Write-Host "`n=== $($Proj.label) deploy (docker compose) ===" -ForegroundColor Cyan
|
|
if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
|
|
if (-not (Test-Path -LiteralPath (Join-Path $root "docker-compose.yml"))) { throw "Missing $root\docker-compose.yml" }
|
|
|
|
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
|
Invoke-Ec2Step "ensure project dir" "sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
|
|
|
|
$files = @(Get-ChildItem -LiteralPath $root -File -Force | Where-Object { $_.Name -ne 'nul' })
|
|
foreach ($f in $files) {
|
|
Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan
|
|
scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
# Config subdirectories ship too, exactly as the edge kind does. Uploading
|
|
# top-level files ONLY was silently wrong for any stack that keeps config in
|
|
# a directory: the compose file arrives, the containers restart, and the
|
|
# config they read is whatever was already on the box. That is worse than a
|
|
# failed deploy, because it reports success - a provisioning directory read
|
|
# at container start (alert rules, datasources, mounted *.php) would never
|
|
# reflect the change you just deployed.
|
|
#
|
|
# Skipped: $JunkDirNames (.git, __pycache__, .pytest_cache, ...) plus
|
|
# anything the project lists in deploy.skipDirs. That list is how a stack
|
|
# protects SERVER-SIDE STATE that happens to share the tree - a data/ holding
|
|
# mailboxes or a time-series database must never be overwritten by whatever
|
|
# the local checkout has (usually nothing, which is the dangerous case).
|
|
# .github is CI config - it belongs in the repo and never on a deploy
|
|
# target. It is not in $JunkDirNames because backups DO want it.
|
|
$skipDirs = @($script:JunkDirNames) + @('.github')
|
|
if ($Proj.deploy -and $Proj.deploy.skipDirs) { $skipDirs += @($Proj.deploy.skipDirs) }
|
|
$dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
|
|
foreach ($d in $dirs) {
|
|
Write-Host " >> uploading $($d.Name)/ (recursive)" -ForegroundColor DarkCyan
|
|
scp -r @SCP_OPTS -i $PEM_KEY $d.FullName "${SSH_TARGET}:$remotePath/"
|
|
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
|
|
}
|
|
|
|
Invoke-Ec2Step "docker compose pull" "cd $remotePath && sudo docker compose pull"
|
|
Invoke-Ec2Step "docker compose up -d" "cd $remotePath && sudo docker compose up -d"
|
|
|
|
Invoke-Ec2PostDeployCleanup -Label $Key
|
|
Write-Host "`n--- [Done] $($Proj.label) deploy finished ---" -ForegroundColor Green
|
|
Write-DeployLocation -Proj $Proj
|
|
}
|
|
|
|
# Pseudo-project "ztokens": not a zconfig entry, no compose stack, and entirely
|
|
# optional. Runs `ztokens -Publish` from a sibling ztokens checkout so a site
|
|
# that charts usage data has something current to ship. Missing checkout, or a
|
|
# failure, warns rather than aborting the rest of the deploy list - it is a
|
|
# nice-to-have refresh, not a deploy step.
|
|
function Invoke-ZTokensPublish {
|
|
Write-Host "`n=== ztokens: refreshing live-usage stats ===" -ForegroundColor Cyan
|
|
$ztokensScript = Join-Path (Split-Path -Parent $PSScriptRoot) "ztokens\ztokens.ps1"
|
|
if (-not (Test-Path -LiteralPath $ztokensScript)) {
|
|
Write-Host " ztokens.ps1 not found at $ztokensScript - skipping." -ForegroundColor Yellow
|
|
return
|
|
}
|
|
# Post-condition, not decoration. This step ran clean for five days while
|
|
# publishing nothing: ztokens.ps1 had no -Publish switch, and as a simple
|
|
# script PowerShell swallowed the unknown parameter into $args rather than
|
|
# failing. The catch below never fired because nothing threw. So the check
|
|
# is not "did it throw" but "did the file actually move".
|
|
$statsFile = Join-Path (Split-Path -Parent $PSScriptRoot) "www\public\token-stats.json"
|
|
$before = if (Test-Path -LiteralPath $statsFile) { (Get-Item -LiteralPath $statsFile).LastWriteTimeUtc } else { [datetime]::MinValue }
|
|
try {
|
|
& $ztokensScript -Publish
|
|
} catch {
|
|
Write-Host " ztokens -Publish failed: $($_.Exception.Message)" -ForegroundColor Yellow
|
|
return
|
|
}
|
|
$after = if (Test-Path -LiteralPath $statsFile) { (Get-Item -LiteralPath $statsFile).LastWriteTimeUtc } else { [datetime]::MinValue }
|
|
if ($after -le $before) {
|
|
Write-Host " WARNING: token-stats.json was not rewritten - the site will ship the old numbers." -ForegroundColor Yellow
|
|
if ($before -eq [datetime]::MinValue) {
|
|
Write-Host " $statsFile does not exist." -ForegroundColor DarkGray
|
|
} else {
|
|
Write-Host (" Still dated {0:yyyy-MM-dd HH:mm} local." -f $before.ToLocalTime()) -ForegroundColor DarkGray
|
|
}
|
|
Write-Host " Run 'ztokens -Publish' by hand to see why." -ForegroundColor DarkGray
|
|
}
|
|
}
|
|
|
|
# ── Dispatch ─────────────────────────────────────────────────────────────────
|
|
|
|
foreach ($key in $Projects) {
|
|
# 'ztokens' matches the tool it runs (ztokens.cmd / ztokens.ps1). The old
|
|
# singular 'ztoken' still works so existing habits and any script that
|
|
# already calls it keep running.
|
|
if ($key -in @('ztokens', 'ztoken')) { Invoke-ZTokensPublish; continue }
|
|
$proj = Get-ZProject -Key $key
|
|
Invoke-DeployGitPull -Proj $proj # no-op unless deploy.gitPull is set
|
|
switch ([string]$proj.kind) {
|
|
"python" { Invoke-PythonDeploy -Key $key -Proj $proj -ChangeNote $Note }
|
|
"vite" { Invoke-ViteDeploy -Key $key -Proj $proj -ChangeNote $Note }
|
|
"nextjs" { Invoke-NextDeploy -Key $key -Proj $proj -ChangeNote $Note }
|
|
"edge" { Invoke-EdgeDeploy -Key $key -Proj $proj }
|
|
"docker" { Invoke-DockerDeploy -Key $key -Proj $proj }
|
|
"static" { Invoke-StaticDeploy -Key $key -Proj $proj }
|
|
default { throw "No deploy handler for kind '$($proj.kind)' (project '$key'). Add an Invoke-<Kind>Deploy function in zdeploy.ps1." }
|
|
}
|
|
}
|
|
# The timestamp goes through Stop-ZTracking as the FinalNote so it lands after
|
|
# the tracking footer and before the trailing blank lines - the last thing on
|
|
# screen, which is the point: scroll to the bottom and you can see how long ago
|
|
# this deployed. Only reached on success; a failed deploy throws out of the loop
|
|
# above, so this never claims a deploy that didn't happen.
|
|
Stop-ZTracking -FinalNote ("Last deployed at {0}" -f (Get-Date -Format "MM/dd/yyyy hh:mm:ss tt"))
|