# Evomedia.net Token Savers — https://github.com/evomedia-net/evo.zscripts
# Created by Kelly Michels · dev@evomedia.net
# Licensed under the MIT License. See LICENSE.
# Version: v1.0.0.0.25

# zdeploy.ps1 — deploy any project defined in zconfig.json to the server.
# Each project runs its own docker compose stack; the handler is picked by the
# project's "kind": python | vite | nextjs | edge | docker | static.
#
# Usage:
#   zdeploy <project> [<project> ...] [-Note "message"]
#   zdeploy -Scan                     # report what is merged but not shipped, then offer to deploy it
#   zdeploy -s -Yes                   # same, unattended (no confirmation prompt)
#   zdeploy -s <project> ...          # scan only these
#   zdeploy all                       # ztokens first, then every project (edge kinds next), stop at first failure
#   zdeploy ztokens                   # refresh the live-usage stats (see below)
#
# Examples:
#   zdeploy viteapp
#   zdeploy pyapp -Note "fix billing banner"
#   zdeploy all -Note "weekly release"
#   zdeploy ztokens evo               # refresh token-stats.json, then ship the site with it
#
# "ztokens" is a pseudo-project, not a zconfig entry: it runs `ztokens -Publish`
# from the sibling ztokens repo, refreshing the token-stats.json the public
# zscripts page charts. `all` runs it first automatically; called standalone,
# list it before a site project (as above) so that project's deploy zip picks
# up the freshly written file.
#
# Flow (python/vite/nextjs): zip source -> free server disk space -> scp up ->
# unzip into remote.path (preserving server-side .env* files and anything in
# deploy.preserve) -> docker compose build + up -> verify the live site reports
# the new build version. Zips are always deleted.
#
# Compose service-name conventions (override with remote.appService):
#   python kind: app service "app", db service "db"
#   nextjs kind: app service "web", db service "db"
#
# Verification (python kind, when there is no scripts/build_version_tool.py):
#   Projects with a "verify" block are checked ON the server via
#   localhost:<port><path> — the only accurate way for stacks that are not
#   published through the edge proxy. Projects with only a "domain" fall back
#   to a Host-header request. Projects with neither are reported as NOT
#   verified rather than passing on the proxy's default vhost.
#
param(
    [Parameter(Position = 0, ValueFromRemainingArguments = $true)]
    [string[]]$Projects = @(),
    [string]$Note = "Build deployed",
    # -Scan / -s: report which projects have work on the default branch that is
    # not live yet, then offer to deploy exactly those. See Get-DeployStatus.
    [Alias('s')][switch]$Scan,
    # Skip the confirmation prompt after a scan. Needed for unattended runs -
    # Read-Host has no answer in a non-interactive shell and would throw.
    [switch]$Yes
)

$ErrorActionPreference = "Stop"
. (Join-Path $PSScriptRoot "ZHelpers.ps1")
Start-ZTracking

$cfg        = Get-ZConfig
$EC2_IP     = $cfg.ec2.ip
$PEM_KEY    = $cfg.ec2.pemKey
$STACK_ROOT = $cfg.ec2.stackRoot
$SSH_TARGET = Get-Ec2Target
$SSH_OPTS   = Get-Ec2SshOpts   # see ZHelpers.ps1 - these are what stop a deploy hanging
$SCP_OPTS   = Get-Ec2ScpOpts   # same, minus -n: scp rejects it with a usage error
$RemoteHome = Get-Ec2Home
$Ec2User    = $cfg.ec2.user

$TempRoot = $cfg.paths.temp
if (-not (Test-Path -LiteralPath $TempRoot)) {
    New-Item -ItemType Directory -Path $TempRoot -Force | Out-Null
}

# ── Scan: what is merged but not shipped ─────────────────────────────────────
# Answers "which projects have work on the default branch that is not live?"
#
# WHAT IT COMPARES
# ----------------
# Every deploy leaves .last_deploy_sha and .last_deploy_utc in the project's
# remote path. The SHA is the real answer: deployed commit versus the current
# default-branch tip, exact regardless of clocks.
#
# .last_deploy_sha only exists from this change onward, so a project that has
# not been deployed since falls back to comparing the tip's COMMIT TIME against
# the deploy time. That is approximate on purpose and is labelled "~" in the
# output: commit time is when the work was authored, not when it merged, so a
# long-lived branch merged today carries an old timestamp and can read as
# already-shipped. The fallback disappears the first time each project deploys.
#
# WHAT IT DOES NOT DO
# -------------------
# It does not judge whether the pending commits change anything shippable - a
# README-only commit still reads as pending. Deploying that is wasteful, not
# wrong, and the alternative (guessing which paths matter per project kind) is
# the sort of cleverness that eventually skips a real change.
function Get-DeployStatus {
    param([string[]]$Keys)

    $cfgLocal = Get-ZConfig
    $rows = @()

    # One ssh for every project rather than one each: this is a status read
    # people will run often, and 15 round trips to answer one question is the
    # difference between a habit and a chore. No $( ) and no embedded double
    # quotes - see the note on Invoke-Ec2Step.
    $parts = @()
    foreach ($k in $Keys) {
        $p = $cfgLocal.projects.$k
        if (-not ($p -and $p.remote -and $p.remote.path)) { continue }
        $rp = $p.remote.path
        $parts += "printf '$k\t'; cat $rp/.last_deploy_sha 2>/dev/null | tr -d '\n'; printf '\t'; cat $rp/.last_deploy_utc 2>/dev/null | tr -d '\n'; printf '\n';"
    }
    $remote = @{}
    if ($parts.Count -gt 0) {
        $prev = $ErrorActionPreference
        $ErrorActionPreference = 'Continue'
        # Deliberately not Invoke-Ec2Step: that prints a step header and throws
        # on failure. A scan wants the output captured, and a box that cannot be
        # reached should degrade to "unknown" rather than abort the report.
        $sshOpts = Get-Ec2SshOpts
        $lines = ssh @sshOpts -i $cfgLocal.ec2.pemKey (Get-Ec2Target) ($parts -join ' ') 2>&1 |
            ForEach-Object { "$_" }
        $ErrorActionPreference = $prev
        foreach ($line in $lines) {
            $f = $line -split "`t"
            if ($f.Count -ge 3) { $remote[$f[0]] = @{ Sha = $f[1].Trim(); Utc = $f[2].Trim() } }
        }
    }

    foreach ($k in $Keys) {
        $p = $cfgLocal.projects.$k
        $row = [ordered]@{ Key = $k; Kind = $p.kind; State = ''; Detail = ''; Ahead = 0 }
        $root = $p.localRoot

        # Not a test for .git in $root: a localRoot may point INTO a repo
        # rather than at its top, when the deployable app is a subdirectory of
        # the checkout. Testing for the folder reported every such project as
        # having no checkout at all. Let git walk up instead.
        $isRepo = $false
        if ($root -and (Test-Path -LiteralPath $root)) {
            $prev = $ErrorActionPreference
            $ErrorActionPreference = 'Continue'
            git -C $root rev-parse --is-inside-work-tree 2>$null | Out-Null
            $isRepo = ($LASTEXITCODE -eq 0)
            $ErrorActionPreference = $prev
        }
        if (-not $isRepo) {
            $row.State = 'no-repo'; $row.Detail = 'no git checkout'
            $rows += [pscustomobject]$row; continue
        }

        Push-Location -LiteralPath $root
        try {
            $prev = $ErrorActionPreference
            $ErrorActionPreference = 'Continue'
            git fetch origin --prune --quiet
            $default = (git symbolic-ref --short refs/remotes/origin/HEAD 2>$null) -replace '^origin/', ''
            if (-not $default) { $default = 'main' }
            $tip = (git rev-parse "origin/$default" 2>$null)
            $tipUtc = (git show -s --format=%cI "origin/$default" 2>$null)
            # Same exclusions as the deploy's own guard, or the scan would
            # report a blocker zdeploy would happily run through: untracked
            # files ship anyway, and build-version.json / CHANGELOG.md are
            # written BY a deploy.
            $dirty = git status --porcelain --untracked-files=no | Where-Object {
                $name = ($_ -replace '^..\s+', '') -replace '^.*/', ''
                @('build-version.json', 'CHANGELOG.md') -notcontains $name
            }
            $ErrorActionPreference = $prev

            if (-not $tip) { $row.State = 'no-repo'; $row.Detail = "no origin/$default"; $rows += [pscustomobject]$row; continue }

            $r = $remote[$k]
            if (-not $r) {
                $row.State = 'unknown'; $row.Detail = 'box unreachable'
            }
            elseif ($r.Sha) {
                if ($r.Sha -eq $tip) { $row.State = 'current'; $row.Detail = $tip.Substring(0, 7) }
                else {
                    $row.State = 'PENDING'
                    $n = (git rev-list --count "$($r.Sha)..origin/$default" 2>$null)
                    if (-not $n -or $LASTEXITCODE -ne 0) { $n = '?' }   # deployed SHA not in this repo's history
                    $row.Ahead = $n
                    $row.Detail = "$n commit(s) since $($r.Sha.Substring(0, [Math]::Min(7, $r.Sha.Length)))"
                }
            }
            elseif (-not $r.Utc) {
                # NOT pending. No stamp means this project has never been
                # deployed by a zdeploy that wrote one - which says nothing
                # about whether it is behind. Calling it pending would have
                # swept edge, the mail server and monitoring into an unattended
                # run on no evidence at all, and edge in particular does not
                # take a speculative deploy well. Deploy it once by name to set
                # the baseline; every scan after that is exact.
                $row.State = 'no-stamp'; $row.Detail = 'no deploy stamp - deploy once by name to baseline it'
            }
            else {
                # Timestamp fallback - approximate, flagged with ~.
                $deployedAt = [datetime]::MinValue
                $ok = [datetime]::TryParse(($r.Utc -replace ' UTC$', ''), [ref]$deployedAt)
                $tipAt = [datetime]::MinValue
                $ok2 = [datetime]::TryParse($tipUtc, [ref]$tipAt)
                if ($ok -and $ok2 -and $tipAt.ToUniversalTime() -gt $deployedAt) {
                    $row.State = 'PENDING'
                    $row.Ahead = '~'
                    $row.Detail = "~ tip $(($tipAt.ToUniversalTime()).ToString('MM-dd HH:mm')) > deploy $($r.Utc -replace ' UTC$','')"
                }
                else {
                    $row.State = 'current'; $row.Detail = "~ deployed $($r.Utc -replace ' UTC$','')"
                }
            }

            if ($dirty -and $row.State -eq 'PENDING') {
                $row.State = 'BLOCKED'
                $row.Detail = "$(@($dirty).Count) uncommitted file(s) - deploy would refuse"
            }
        }
        finally { Pop-Location }

        $rows += [pscustomobject]$row
    }
    return $rows
}

# A bare `zdeploy -s` means "look at everything", so it must not fall into the
# usage block below.
if ($Scan -and $Projects.Count -eq 0) { $Projects = @(Get-ZProjectKeys) }

if ($Projects.Count -eq 0) {
    $keys = (Get-ZProjectKeys) -join ', '
    Write-Host ""
    Write-Host "Usage: zdeploy <project> [<project> ...] | all | ztokens  [-Note `"message`"]" -ForegroundColor Yellow
    Write-Host "  Projects in zconfig.json: $keys" -ForegroundColor Gray
    Write-Host "  'all' deploys everything (edge kinds first) and stops at the first failure." -ForegroundColor Gray
    Write-Host "  -Scan / -s reports which projects have merged work that is not live, then offers to deploy just those." -ForegroundColor Gray
    Write-Host "            Add -Yes to skip the confirmation prompt. Edge still ships first." -ForegroundColor Gray
    Write-Host "  'ztokens' refreshes the live-usage stats published to the zscripts page." -ForegroundColor Gray
    Stop-ZTracking; exit 1
}

# Tolerate switch-style args (zdeploy -myproject) from muscle memory.
$Projects = @($Projects | ForEach-Object { $_.TrimStart('-') })

if ($Projects -contains 'all') {
    # 'ztokens' first so any site project deployed below picks up fresh stats.
    $Projects = @('ztokens') + @(Get-ZProjectKeys)
}

# Edge kinds first, however the list was produced. This is a correctness
# property, not a convenience of 'all': the proxy has to route before the apps
# behind it ship, or there is a window where a new app is live behind stale
# routing. `zdeploy evo edge` reads as "these two, edge included" and used to
# do the risky order, because this sort only ran for 'all'.
# Order within each group is preserved, so an intentional sequence still holds
# — notably `zdeploy ztokens evo`, where ztokens must still precede evo.
# Scan runs BEFORE the edge-first sort below, so whatever it selects still gets
# ordered by that rule - the proxy ships before the apps behind it, exactly as
# a hand-typed list would.
if ($Scan) {
    Write-Host "`n=== zdeploy -Scan: what is merged but not shipped ===" -ForegroundColor Cyan
    $status = Get-DeployStatus -Keys @($Projects | Where-Object { $_ -ne 'ztokens' })

    Write-Host ""
    foreach ($r in $status) {
        $colour = switch ($r.State) {
            'PENDING' { 'Yellow' }
            'BLOCKED' { 'Red' }
            'no-stamp' { 'DarkYellow' }
            'current' { 'DarkGray' }
            default   { 'DarkYellow' }
        }
        Write-Host ("  {0,-14} {1,-8} {2,-9} {3}" -f $r.Key, $r.Kind, $r.State, $r.Detail) -ForegroundColor $colour
    }

    $pending = @($status | Where-Object { $_.State -eq 'PENDING' })
    $blocked = @($status | Where-Object { $_.State -eq 'BLOCKED' })
    $unknown = @($status | Where-Object { $_.State -eq 'unknown' })
    $nostamp = @($status | Where-Object { $_.State -eq 'no-stamp' })
    Write-Host ""
    Write-Host ("  {0} pending, {1} blocked, {2} no-stamp, {3} unknown, {4} current" -f `
        $pending.Count, $blocked.Count, $nostamp.Count, $unknown.Count,
        @($status | Where-Object { $_.State -eq 'current' }).Count) -ForegroundColor Gray

    if ($blocked.Count -gt 0) {
        Write-Host "  Blocked projects are NOT deployed - commit or stash them, then re-run." -ForegroundColor Red
    }
    if ($nostamp.Count -gt 0) {
        Write-Host "  No-stamp projects are NOT selected - deploy each once by name to establish a baseline." -ForegroundColor DarkYellow
    }
    if ($unknown.Count -gt 0) {
        # Silence here would read as "nothing to do", which is the one thing an
        # unreachable box does not mean.
        Write-Host "  Unknown = the box did not answer for that project; its state is NOT 'current'." -ForegroundColor DarkYellow
    }

    if ($pending.Count -eq 0) {
        Write-Host "`n  Nothing to deploy.`n" -ForegroundColor Green
        Stop-ZTracking; exit 0
    }

    Write-Host ""
    if (-not $Yes) {
        # Two different ways a prompt can have nobody to answer it, and they
        # fail differently:
        #   - a -NonInteractive host: Read-Host THROWS. Caught below.
        #   - redirected stdin (a pipe, a scheduled task, powershell.exe launched
        #     from another shell): Read-Host does NOT throw - it BLOCKS, waiting
        #     on a pipe that never answers. The first scan run this way sat for
        #     ten minutes with its table already printed but withheld behind the
        #     blocked pipeline. [Environment]::UserInteractive is $true in both
        #     cases, so it cannot be the test; IsInputRedirected can.
        if ([Console]::IsInputRedirected) {
            Write-Host "  stdin is not a terminal - cannot prompt. Re-run with -Yes to deploy these $($pending.Count).`n" -ForegroundColor Yellow
            Stop-ZTracking; exit 0
        }
        $answer = $null
        try { $answer = Read-Host "  Deploy these $($pending.Count)? [y/N]" }
        catch {
            Write-Host "  Non-interactive shell - cannot prompt. Re-run with -Yes to deploy these $($pending.Count).`n" -ForegroundColor Yellow
            Stop-ZTracking; exit 0
        }
        if ($answer -notmatch '^(y|yes)$') {
            Write-Host "  Aborted. Nothing deployed.`n" -ForegroundColor Yellow
            Stop-ZTracking; exit 0
        }
    }
    $Projects = @($pending | ForEach-Object { $_.Key })
}

$requested = @($Projects)
$edgeKeys  = @($Projects | Where-Object { $cfg.projects.$_.kind -eq 'edge' })
$restKeys  = @($Projects | Where-Object { $cfg.projects.$_.kind -ne 'edge' })
$Projects  = @($edgeKeys + $restKeys)

if ($Projects.Count -gt 1) {
    # Say so when the order changed, so the reordering is never silent.
    $note = if (($requested -join ',') -ne ($Projects -join ',')) { "  (edge first)" } else { "" }
    Write-Host "Deploying: $($Projects -join ', ')$note" -ForegroundColor Cyan
}

# The bash that stamps what just shipped. The timestamp has always been
# written; the SHA is what lets -Scan answer exactly rather than by clock
# comparison. Omitted rather than faked when the checkout is not a git repo -
# scan falls back to the timestamp, and a wrong SHA would be worse than none.
function Get-RecordDeployBash {
    param(
        [Parameter(Mandatory)]$Proj,
        [Parameter(Mandatory)][string]$RemotePath,
        # Optional: the edge, static and docker paths upload no zip, so there
        # is nothing to remove - but they still ship, so they still stamp.
        [string]$ZipName = ''
    )
    $sha = ''
    $root = $Proj.localRoot
    if ($root -and (Test-Path -LiteralPath (Join-Path $root '.git'))) {
        $prev = $ErrorActionPreference
        $ErrorActionPreference = 'Continue'
        $sha = (git -C $root rev-parse HEAD 2>$null)
        if ($LASTEXITCODE -ne 0) { $sha = '' }
        $ErrorActionPreference = $prev
    }
    $cmd = "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $RemotePath/.last_deploy_utc > /dev/null"
    # 40 hex characters, so it needs no quoting in the remote command.
    if ($sha) { $cmd += " && printf '%s' $sha | sudo tee $RemotePath/.last_deploy_sha > /dev/null" }
    if ($ZipName) { $cmd += " && rm -f $RemoteHome/$ZipName" }
    return $cmd
}

function Get-DeployZipName {
    param([string]$Key, $Proj)
    if ($Proj.deploy -and $Proj.deploy.zipName) { return $Proj.deploy.zipName }
    return "${Key}Deploy.zip"
}

# Pre-upload cleanup: remove stale deploy zips, prune docker, truncate big logs,
# fail if under 1.5 GB free.
function Invoke-Ec2PreflightCleanup {
    param([string[]]$ExtraZipsToRemove = @())
    Write-Host "`n--- [Preflight] Freeing disk space on the server ---" -ForegroundColor Cyan
    $rmZipsClause = if ($ExtraZipsToRemove.Count -gt 0) { "rm -f $($ExtraZipsToRemove -join ' ')" } else { "true" }
    $preflightCmd = @(
        "echo '--- df / before cleanup ---'",
        "df -h /",
        "echo '--- removing stale deploy artifacts ---'",
        $rmZipsClause,
        "echo '--- pruning docker build cache + dangling images + stopped containers ---'",
        "sudo docker container prune -f >/dev/null 2>&1 || true",
        "sudo docker builder prune -f >/dev/null 2>&1 || true",
        "sudo docker image prune -af >/dev/null 2>&1 || true",
        "echo '--- truncating large container logs ---'",
        "sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true",
        "echo '--- df / after cleanup ---'",
        "df -h /",
        "avail_mb=`$(df --output=avail -BM / | tail -n 1 | tr -dc 0-9)",
        "[ -z `"`$avail_mb`" ] && avail_mb=0",
        "echo available_mb=`$avail_mb",
        "if [ `"`$avail_mb`" -lt 1500 ]; then echo 'ERROR: less than 1.5 GB free on /. Grow the root volume or run: sudo docker system prune -af' >&2; exit 11; fi"
    ) -join '; '
    # Also through the wrapper (#119): the out-of-space branch above writes its
    # ERROR to stderr and exits 11, so a bare ssh would surface a
    # NativeCommandError instead of the actionable message below - exactly when
    # the operator most needs to be told what to do. -FailHint keeps it.
    Invoke-Ec2Step "server pre-flight cleanup" $preflightCmd `
        -FailHint "Root volume too full (need ~1.5 GB free, ideally 3+). Grow it or run: sudo docker system prune -af"
}

# Post-deploy cleanup: prune build cache and dangling images created during this deploy.
# Containers/volumes still in use by the running stack are NOT touched.
function Invoke-Ec2PostDeployCleanup {
    param([string]$Label = "post-deploy")
    Write-Host "`n--- [Post-deploy] Reclaiming disk space ($Label) ---" -ForegroundColor Cyan
    $cmd = @(
        "sudo docker container prune -f >/dev/null 2>&1 || true",
        "sudo docker builder prune -f >/dev/null 2>&1 || true",
        "sudo docker image prune -af >/dev/null 2>&1 || true",
        "sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true",
        "df -h /"
    ) -join '; '
    ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $cmd
    if ($LASTEXITCODE -ne 0) {
        Write-Host "  Post-deploy cleanup returned non-zero exit ($LASTEXITCODE); continuing." -ForegroundColor DarkYellow
    }
}

function Send-DeployZip {
    param([string]$LocalZip, [string]$ZipName)
    scp @SCP_OPTS -i $PEM_KEY $LocalZip "${SSH_TARGET}:$RemoteHome/"
    if ($LASTEXITCODE -ne 0) {
        throw "SCP upload failed (exit $LASTEXITCODE). Read scp's own output above: a usage block means bad arguments, not disk. If it is genuinely full, clear the stale archive: rm -f $RemoteHome/$ZipName"
    }
}

function Invoke-RemoteUnzip {
    param([string]$ZipName, [string]$DestPath)
    $bash = 'test -f {2}/{0} || {{ echo "missing {2}/{0}"; exit 2; }}; unzip -t {2}/{0} || exit 3; unzip -o {2}/{0} -d {1}; uc=$?; if [ $uc -gt 1 ]; then exit $uc; fi; exit 0' -f $ZipName, $DestPath, $RemoteHome
    Invoke-Ec2Step "unzip $ZipName" $bash
}

# ── Operator-file preservation (issue #2, hardened in #107) ──────────────────
# Deploys replace the project directory wholesale, which used to destroy every
# operator-managed file except ./.env. These helpers preserve all .env* files
# at the project root PLUS any paths listed in deploy.preserve (files or
# directories), by tarring them to the home dir before the wipe and extracting
# them back after the unzip. Server-side copies win over anything shipped in
# the zip — the same semantics ./.env always had.
#
# WHY THE ARCHIVE IS TIMESTAMPED AND NEVER DELETED (#107)
# -------------------------------------------------------
# This used to write one fixed preserve_<key>.tgz, and preserve's FIRST action
# was `rm -f` on it. That is the opposite of safe. The window between
# `sudo rm -rf` on the project directory and the restore step is the only time
# the tarball is the sole copy of the production secrets - and an interrupted
# run (dropped ssh, exit 255) stops exactly there, leaving a perfect backup
# behind. The next run then deleted that backup before doing anything else,
# tarred a directory that no longer had the files, and reported success.
# `2>/dev/null; true` on the tar is what made it silent.
#
# That destroyed civilcode's production deploy/.env on 2026-08-30. The site
# survived only because the running container still held its environment; a
# restart would have made the loss permanent.
#
# So, three independent changes, any one of which would have prevented it:
#
#   1. Each run writes its own preserve_<key>_<stamp>.tgz and restore no
#      longer deletes it. Nothing removes an archive that has not been
#      superseded - retention below prunes old ones instead.
#   2. Preserve first extracts any earlier archives with `tar -k`, which fills
#      in files a previous interrupted run lost WITHOUT overwriting anything
#      currently on disk. A hand-repaired .env therefore wins over the stale
#      copy in the archive.
#   3. Preserve refuses to continue if it captured nothing while an earlier
#      archive for the same key did have contents. Capturing zero files is
#      normal for a project with no operator files (edge, gitea, landing) and
#      catastrophic for one that has them; the prior archive is what tells the
#      difference.
#
# One stamp per zdeploy process, so preserve and restore agree on the filename
# without threading it through every call site.
$script:PreserveStamp = Get-Date -Format 'yyyyMMdd-HHmmss'
$script:PreserveKeep  = 5

function Get-PreserveTarball {
    param([string]$Key)
    "$RemoteHome/preserve_${Key}_$($script:PreserveStamp).tgz"
}

function Save-OperatorFiles {
    param([string]$Key, $Proj, [string]$RemotePath)
    $paths = @('.env*')
    if ($Proj.deploy -and $Proj.deploy.preserve) { $paths += @($Proj.deploy.preserve) }
    $spec = $paths -join ' '
    $tarball = Get-PreserveTarball -Key $Key
    $list    = $tarball -replace '\.tgz$', '.list'
    $glob    = "$RemoteHome/preserve_${Key}_*"
    $drop    = $script:PreserveKeep + 1

    # NOTE: no embedded double quotes or $( ) here - PowerShell 5.1 strips
    # embedded double quotes when passing args to ssh.exe, silently corrupting
    # the remote command, and $( ) would be evaluated locally. Remote shell
    # variables are backtick-escaped so PowerShell leaves them alone. Globs
    # expand remotely; tar's nonzero exit for missing paths is swallowed, but
    # an empty capture is NOT (see the guard below).
    $bash =
        "cd $RemotePath || exit 9; " +
        "for t in ${glob}.tgz; do [ -e `$t ] && tar -xzkf `$t -C $RemotePath 2>/dev/null; done; true; " +
        "tar -czf $tarball $spec 2>/dev/null; " +
        "tar -tzf $tarball > $list 2>/dev/null; " +
        "if [ ! -s $list ]; then " +
          "for p in ${glob}.list; do " +
            "if [ -s `$p ] && [ `$p != $list ]; then " +
              "echo PRESERVE CAPTURED NOTHING BUT AN EARLIER ARCHIVE HAS FILES; exit 8; " +
            "fi; " +
          "done; " +
        "fi; " +
        "ls -1t ${glob}.tgz 2>/dev/null | tail -n +$drop | xargs -r rm -f; " +
        "ls -1t ${glob}.list 2>/dev/null | tail -n +$drop | xargs -r rm -f; " +
        "exit 0"

    Invoke-Ec2Step "preserve operator files ($spec)" $bash `
        -FailHint "Refusing to wipe $RemotePath - see $glob.tgz on the server."
}

function Restore-OperatorFiles {
    param([string]$Key, [string]$RemotePath)
    $tarball = Get-PreserveTarball -Key $Key
    # Overwrites, deliberately: server-side operator files beat whatever the
    # zip shipped. The archive is left in place - see the header.
    Invoke-Ec2Step "restore operator files" "test -f $tarball && tar -xzf $tarball -C $RemotePath; true"
}

# ── Deploy verification (build-version match, not just HTTP 200 — a 200 can be
#    a stale cached build; the version match proves the new build is live) ─────

function Wait-VerifyStaticBuild {
    param([string]$Key, $Proj, [object]$PreZipBuildState)
    if (-not $PreZipBuildState) {
        Write-Host "`n--- [$Key version] SKIPPED (no local build-version.json - see 'Enabling deploy verification' in README) ---" -ForegroundColor DarkYellow
        return
    }
    # Expect the COMMITTED stamp, not +1: builds no longer self-bump (a
    # prebuild hook incremented inside the image, so the served version
    # matched no commit and every build dirtied the tree). The counter now
    # advances deliberately — one version bump per release — so "is the
    # build I just packed live?" means an exact match.
    $expectedLabel = Get-LabelFromBuildJsonObj $PreZipBuildState
    # A stamp we cannot read is not a version to check against. Comparing an
    # unreadable local label to an unreadable remote one is how a verification
    # once passed while the container served anything it liked, so refuse to
    # run rather than run a comparison that cannot fail.
    if ([string]::IsNullOrWhiteSpace($expectedLabel)) {
        Write-Host "`n--- [$Key version] NOT VERIFIED - build-version.json is present but unreadable ---" -ForegroundColor Yellow
        Write-Host "  Got: $(($PreZipBuildState | ConvertTo-Json -Compress -Depth 4))" -ForegroundColor DarkGray
        Write-Host "  Expected one of: {`"version`":`"v1.0.0.0.0`"} | {major,rc,beta,alpha,build} | {productVersion,buildNumber}" -ForegroundColor DarkGray
        return
    }
    Write-Host "`n--- [$Key] Live build verification (expect $expectedLabel) ---" -ForegroundColor Cyan
    $containerName = $Proj.remote.containerName
    $deadline = (Get-Date).AddSeconds(45)
    while ((Get-Date) -lt $deadline) {
        try {
            $r = $null
            if ($containerName) {
                # build-version.json may be blocked from external requests by the edge
                # proxy; read it inside the running container instead.
                $raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET `
                    "sudo docker exec $containerName cat /usr/share/nginx/html/build-version.json 2>/dev/null"
                if ($raw) { $r = $raw | ConvertFrom-Json -ErrorAction Stop }
            } else {
                $headers = @{}
                if ($Proj.domain) { $headers['Host'] = $Proj.domain }
                $r = Invoke-RestMethod -Uri "http://$EC2_IP/build-version.json" -Headers $headers -TimeoutSec 10 -ErrorAction Stop
            }
            if ($r) {
                $remoteLabel = Get-LabelFromBuildJsonObj $r
                if ($remoteLabel -and $remoteLabel -eq $expectedLabel) {
                    Write-Host "  PASS - live build $remoteLabel matches expected." -ForegroundColor Green
                    return
                }
                Write-Host "  Live build is $remoteLabel, expected $expectedLabel - waiting..." -ForegroundColor DarkYellow
            }
        } catch {
            Write-Host "  Container not ready yet - waiting..." -ForegroundColor DarkGray
        }
        Start-Sleep -Seconds 3
    }
    Write-Host "  WARNING: live build did not match $expectedLabel within 45s (upload or Docker build may have failed, or a stale build is cached)." -ForegroundColor Yellow
}

function Wait-VerifyApiBuild {
    param([string]$Key, $Proj, [string]$ExpectedLabel, [int]$TimeoutSec = 60)
    Write-Host "`n--- [$Key] Live build verification (expect $ExpectedLabel) ---" -ForegroundColor Cyan
    # deploy.verifyHost (or domain) decides which edge vhost may be asked;
    # both are consumed inside Get-VerifyAttempts now, where "no host at all"
    # excludes the edge channel entirely rather than defaulting to whatever
    # vhost the proxy serves (#101). verifyHost exists for a domain retired
    # ahead of its replacement - a takedown once had a project's public host
    # answering 410 while the app was healthy; no project sets it today.
    # Every retry walks the channels in trust order - docker-network exec,
    # then localhost port, then (only with a Host to route by) the edge.
    # The choice used to be made ONCE, before the loop, by probing each
    # channel - but the probes ran at the exact moment step [5] had
    # restarted the app, so both good channels were briefly down and the
    # whole window was spent on the edge. For a project with no domain that
    # meant the default vhost: one project's check read a DIFFERENT
    # project's build number, twice in a single day (#101). Re-resolving per
    # retry means the right channel is used the moment the app is back.
    #
    # The port channel still counts only when the body carries a version:
    # one project's verify path is a JWKS endpoint - real, healthy, and no
    # version in it - so it falls through to the edge (it has a domain),
    # same as it always did.
    $execCmd  = Get-ServerSideVersionCommand -Proj $Proj
    $attempts = Get-VerifyAttempts -Proj $Proj -ExecCmd $execCmd
    $TimeoutSec = Get-VerifyTimeout -Proj $Proj -DefaultSec $TimeoutSec
    if ($attempts.Count -eq 0) {
        # No trustworthy channel exists: no viaProxy, no port, no host to
        # route an edge request by. Asking the edge anyway can only reach
        # the DEFAULT vhost - a different product - and a check that can
        # only ever read someone else's number is worse than no check.
        Write-Host "  SKIPPED: no way to verify this project without reading the wrong vhost - configure verify.viaProxy/port, or a domain (#101)." -ForegroundColor Yellow
        return $false
    }
    Write-Host "  Channels, in order: $(($attempts | ForEach-Object { $_.Label }) -join '; ')" -ForegroundColor DarkGray

    $sawVersion = $false
    $deadline = (Get-Date).AddSeconds($TimeoutSec)
    while ((Get-Date) -lt $deadline) {
        foreach ($attempt in $attempts) {
            $r = $null
            try {
                switch ($attempt.Kind) {
                    'exec' {
                        $raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $execCmd
                        $r = ($raw | Out-String).Trim() | ConvertFrom-Json -ErrorAction Stop
                    }
                    'port' {
                        $raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "curl -s -m 8 http://localhost:$($attempt.Port)$($attempt.Path)"
                        $r = ($raw | Out-String).Trim() | ConvertFrom-Json -ErrorAction Stop
                    }
                    'edge' {
                        $edgeHeaders = @{ 'Host' = $attempt.HostHeader }
                        $r = Invoke-RestMethod -Uri "http://$EC2_IP/api/build-version" -Headers $edgeHeaders -TimeoutSec 10 -ErrorAction Stop
                    }
                }
            } catch {
                continue  # channel not ready; the next one gets its turn
            }
            if ($null -eq $r) { continue }
            # Two field names in the fleet: some apps answer build_version
            # on /api/build-version, others answer version on /health. Both
            # are "the build that is live", so accept either rather than
            # making every app rename its own field.
            $live = if ($r.build_version) { [string]$r.build_version } elseif ($r.version) { [string]$r.version } else { $null }
            if (-not $live) { continue }  # answered, but not about versions (JWKS etc.)
            $sawVersion = $true
            if ($live -eq $ExpectedLabel) {
                Write-Host "  PASS - live build $live matches expected ($($attempt.Label))." -ForegroundColor Green
                return $true
            }
            Write-Host "  Live build is $live via $($attempt.Label), expected $ExpectedLabel - waiting..." -ForegroundColor DarkYellow
            break  # one wrong-version read this pass is enough; retry after the sleep
        }
        Start-Sleep -Seconds 3
    }
    # Say which failure this actually was: a version that never matched is a
    # stale/failed build; channels that never answered is "could not verify",
    # and pretending otherwise is how a warning gets ignored.
    if ($sawVersion) {
        Write-Host "  WARNING: live build did not match $ExpectedLabel within ${TimeoutSec}s (upload or Docker build may have failed, or a stale build is cached)." -ForegroundColor Yellow
    } else {
        Write-Host "  WARNING: could not verify within ${TimeoutSec}s - no channel answered with a version (app may still be starting; raise verify.timeoutSeconds if this project boots slowly)." -ForegroundColor Yellow
    }
    return $false
}

# Verify a deploy by calling the app ON the server (localhost:<port>). Works
# for stacks that are not published through the edge proxy or whose host port
# is closed to the internet — hitting http://<ec2-ip>/ for those just answers
# from whatever vhost the proxy serves by default, which is a false PASS.
#
# Configure per project in zconfig.json:
#   "verify": { "port": 8005, "path": "/health", "expect": "\"status\":\"ok\"" }
# port is required; path defaults to "/", expect is an optional substring.
function Test-DeployHealth {
    param([string]$Key, $Proj, [int]$TimeoutSec = 60)

    $port   = [int]$Proj.verify.port
    $path   = if ($Proj.verify.path) { [string]$Proj.verify.path } else { "/" }
    $expect = [string]$Proj.verify.expect

    Write-Host "`n--- [$Key] Health check (on server: localhost:$port$path) ---" -ForegroundColor Cyan
    $deadline = (Get-Date).AddSeconds($TimeoutSec)
    while ((Get-Date) -lt $deadline) {
        # -L: an app whose "/" redirects (e.g. Next.js "/" -> "/login") answers a
        # 307 whose body is a few bytes or empty, which reads as "not ready".
        # Follow to the page that actually renders before judging.
        $raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "curl -sL -m 8 http://localhost:$port$path"
        $body = ($raw | Out-String).Trim()
        if ($LASTEXITCODE -eq 0 -and $body) {
            if (-not $expect -or $body.Contains($expect)) {
                # A /health endpoint returns a line of JSON; an app page returns
                # kilobytes of HTML. Match on the whole body, but only print
                # enough to recognise it — the rest is unreadable in a log.
                $shown = if ($body.Length -gt 200) { $body.Substring(0, 200) + "... ($($body.Length) bytes)" } else { $body }
                Write-Host "  PASS - $shown" -ForegroundColor Green
                return $true
            }
            Write-Host "  Responding but '$expect' not found - waiting..." -ForegroundColor DarkYellow
        } else {
            Write-Host "  Not ready yet - waiting..." -ForegroundColor DarkGray
        }
        Start-Sleep -Seconds 3
    }
    Write-Host "  WARNING: no healthy response from localhost:$port$path within ${TimeoutSec}s." -ForegroundColor Yellow
    return $false
}

# Where the thing just deployed can be reached. A project published through the
# edge proxy has a domain; one that is not still has somewhere to point at, and
# saying nothing is the least useful option — an internal service is exactly the
# case where "where did that land?" is hardest to answer from memory. Falls back
# through what the project actually declares, and prints nothing if it declares
# none of it.
function Write-DeployLocation {
    param($Proj, [int]$Pad = 0)
    $label = "Site:".PadRight([Math]::Max(5, $Pad))
    if ($Proj.domain) {
        Write-Host "$label https://$($Proj.domain)" -ForegroundColor Yellow
        return
    }
    # No public route: give the server-local endpoint the deploy just verified.
    $port = if ($Proj.verify -and $Proj.verify.port) { [int]$Proj.verify.port }
            elseif ($Proj.ports -and $Proj.ports.prod) { [int]$Proj.ports.prod }
            else { 0 }
    if ($port -le 0) { return }
    $path = if ($Proj.verify -and $Proj.verify.path) { [string]$Proj.verify.path } else { "" }
    Write-Host "$label http://127.0.0.1:$port$path  (on the server; no public domain)" -ForegroundColor Yellow
}

# ── Kind handlers ────────────────────────────────────────────────────────────

function Invoke-PythonDeploy {
    param([string]$Key, $Proj, [string]$ChangeNote)

    $DeployStart = Get-Date
    $prevLoc     = Get-Location
    $root        = $Proj.localRoot
    $remotePath  = $Proj.remote.path
    $composeDir  = if ($Proj.remote.composeDir) { $Proj.remote.composeDir } else { $remotePath }
    $appSvc      = if ($Proj.remote.appService) { $Proj.remote.appService } else { "app" }
    $zipName     = Get-DeployZipName -Key $Key -Proj $Proj
    $zipLocal    = Join-Path $TempRoot $zipName
    $BuildVersion = $null
    $versionTool = Join-Path $root "scripts\build_version_tool.py"
    $hasVersionTool = Test-Path -LiteralPath $versionTool

    try {
        if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
        Set-Location -LiteralPath $root

        Write-Host "`n=== $($Proj.label) deploy (python) ===" -ForegroundColor Cyan
        Write-Host "Local zip: $zipLocal" -ForegroundColor DarkGray

        Write-Host "`n--- [1] Zipping $($Proj.label) ---" -ForegroundColor Cyan
        Get-ChildItem -LiteralPath $root -Directory -Recurse -Filter "__pycache__" -ErrorAction SilentlyContinue |
            Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
        New-ProjectArchive -SourcePath $root -DestinationZip $zipLocal -TopLevelExclude (Get-ArchiveExcludes -Project $Proj)

        Invoke-Ec2PreflightCleanup -ExtraZipsToRemove @("$RemoteHome/$zipName")

        Write-Host "`n--- [2] Uploading zip ---" -ForegroundColor Cyan
        Send-DeployZip -LocalZip $zipLocal -ZipName $zipName

        Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
        Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
        Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
        Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
        Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
        Invoke-Ec2Step "replace project directory" "sudo rm -rf $remotePath && sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
        Invoke-RemoteUnzip -ZipName $zipName -DestPath $remotePath
        Restore-OperatorFiles -Key $Key -RemotePath $remotePath
        Invoke-Ec2Step "require compose directory" "test -d $composeDir"
        Invoke-Ec2Step "docker compose build $appSvc" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose build $appSvc"
        Invoke-Ec2Step "docker compose up -d" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose up -d"
        Invoke-Ec2Step "record deploy time; remove remote zip" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath -ZipName $zipName)

        if ($hasVersionTool) {
            Write-Host "`n--- [4] Incrementing build version ---" -ForegroundColor Cyan
            $BumpCmd = "cd $composeDir && sudo docker compose exec -T $appSvc python scripts/build_version_tool.py bump"
            for ($attempt = 1; $attempt -le 5; $attempt++) {
                $output = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $BumpCmd
                if ($LASTEXITCODE -eq 0 -and $output) {
                    $BuildVersion = ($output | Select-Object -Last 1).ToString().Trim()
                    break
                }
                Write-Host "  Attempt $attempt failed, retrying in 3s..." -ForegroundColor DarkYellow
                Start-Sleep -Seconds 3
            }
            if (-not $BuildVersion) { throw "Build version bump failed after 5 attempts" }

            # The local stamp is deliberately NOT mirrored back. Writing it
            # left build-version.json dirty after every deploy, and committing
            # that hit branch protection ("Changes must be made through a pull
            # request") — so each deploy either tripped the NEXT deploy's
            # clean-tree guard or bypassed the rule. Neither is acceptable as
            # routine behaviour.
            #
            # The repo file now records the STAGE baseline only (it changes on
            # a stage bump, through a normal PR). The live build number lives
            # in the container, is written to .build_version below, and is
            # proven by the /api/build-version check — which is the thing that
            # actually establishes what is deployed.
            Invoke-Ec2Step "record the live build number" "echo '$BuildVersion' | sudo tee $remotePath/.build_version > /dev/null"
            $changelogTool = Join-Path $root "scripts\build_changelog_tool.py"
            if (Test-Path -LiteralPath $changelogTool) {
                if ([string]::IsNullOrWhiteSpace($ChangeNote)) { $ChangeNote = "Build deployed" }
                python $changelogTool append --version $BuildVersion --note "$ChangeNote" | Out-Null
            }

            Write-Host "`n--- [5] Restarting app to pick up new version ---" -ForegroundColor Cyan
            # Through Invoke-Ec2Step, not a bare ssh: `docker compose restart`
            # writes " Container <name>  Restarting" to STDERR as ordinary
            # progress, and under ErrorActionPreference='Stop' PS 5.1 turns any
            # native stderr line into a terminating NativeCommandError whatever
            # the exit code. That threw here on a deploy that had fully
            # succeeded - and it threw BEFORE Wait-VerifyApiBuild, so the step
            # that proves what is actually deployed never ran (#119). The
            # wrapper flattens stderr and judges by exit code alone, and throws
            # on non-zero itself, so the hand-written check is gone with it.
            Invoke-Ec2Step "restart $appSvc to pick up the new build" `
                "cd $composeDir && sudo COMPOSE_BAKE=false docker compose restart $appSvc" `
                -FailHint "App restart after the build bump failed."

            Wait-VerifyApiBuild -Key $Key -Proj $Proj -ExpectedLabel $BuildVersion -TimeoutSec 30 | Out-Null

            # Only now: the tag is a claim about what is RUNNING, so it
            # is written after the live build has been proven, never before.
            New-DeployTag -Proj $Proj -Version $BuildVersion -Note $ChangeNote
        } elseif ($Proj.verify -and $Proj.verify.port) {
            Test-DeployHealth -Key $Key -Proj $Proj -TimeoutSec 60 | Out-Null
        } elseif ($Proj.domain) {
            Write-Host "`n--- [4] Basic reachability check (no build_version_tool - see 'Enabling deploy verification' in README) ---" -ForegroundColor Cyan
            $headers = @{ 'Host' = $Proj.domain }
            $deadline = (Get-Date).AddSeconds(30)
            $up = $false
            while ((Get-Date) -lt $deadline) {
                Start-Sleep -Seconds 3
                try {
                    $resp = Invoke-WebRequest -Uri "http://$EC2_IP/" -Headers $headers -UseBasicParsing -TimeoutSec 8 -ErrorAction Stop
                    if ($resp.StatusCode -lt 500) { $up = $true; break }
                } catch { Write-Host "  App not ready yet - waiting..." -ForegroundColor DarkGray }
            }
            if ($up) { Write-Host "  App is responding." -ForegroundColor Green }
            else { Write-Host "  WARNING: app did not respond within 30s." -ForegroundColor Yellow }
        } else {
            # No domain to send as a Host header and no "verify" block: a request
            # to http://<ec2-ip>/ would be answered by the proxy's default vhost,
            # so it proves nothing about THIS app. Say so instead of faking a PASS.
            Write-Host "`n--- [4] Deploy finished - NOT verified ---" -ForegroundColor Yellow
            Write-Host "  No 'domain' and no 'verify' block in zconfig.json for '$Key'," -ForegroundColor Yellow
            Write-Host "  so there is no way to confirm the new build is live." -ForegroundColor Yellow
            Write-Host '  Add to the project: "verify": { "port": <hostPort>, "path": "/health" }' -ForegroundColor Gray
        }

        Invoke-Ec2PostDeployCleanup -Label $Key

        $Elapsed    = (Get-Date) - $DeployStart
        $ElapsedStr = "{0:mm\:ss}" -f $Elapsed
        Write-Host "`n--- [Done] $($Proj.label) deployed! ---" -ForegroundColor Green
        Write-DeployLocation -Proj $Proj
        if ($BuildVersion) { Write-Host "Build Version: $BuildVersion" -ForegroundColor Magenta }
        Write-Host "Change Note: $ChangeNote" -ForegroundColor Cyan
        Write-Host "Deploy Time: $ElapsedStr ($([math]::Round($Elapsed.TotalSeconds))s)" -ForegroundColor DarkGray
    }
    finally {
        if (Test-Path -LiteralPath $zipLocal) {
            try { Remove-Item -LiteralPath $zipLocal -Force -ErrorAction Stop } catch { }
        }
        Set-Location -LiteralPath $prevLoc
    }
}

function Invoke-ViteDeploy {
    param([string]$Key, $Proj, [string]$ChangeNote)

    $DeployStart = Get-Date
    $prevLoc     = Get-Location
    $root        = $Proj.localRoot
    $remotePath  = $Proj.remote.path
    $zipName     = Get-DeployZipName -Key $Key -Proj $Proj
    $zipLocal    = Join-Path $TempRoot $zipName
    $preZipBuild = $null

    try {
        if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
        Set-Location -LiteralPath $root

        Write-Host "`n=== $($Proj.label) deploy (vite/static) ===" -ForegroundColor Cyan
        Write-Host "Local zip: $zipLocal" -ForegroundColor DarkGray

        Write-Host "`n--- [1] Zipping site ---" -ForegroundColor Cyan
        $preZipBuild = Read-JsonBuildVersion -FilePath (Join-Path $root "build-version.json")
        if ($preZipBuild) {
            Write-Host "  Pre-zip build label: $(Get-LabelFromBuildJsonObj $preZipBuild) (verification expects this exact label)" -ForegroundColor Gray
        }
        New-ProjectArchive -SourcePath $root -DestinationZip $zipLocal -TopLevelExclude (Get-ArchiveExcludes -Project $Proj)

        Invoke-Ec2PreflightCleanup -ExtraZipsToRemove @("$RemoteHome/$zipName")

        Write-Host "`n--- [2] Uploading zip ---" -ForegroundColor Cyan
        Send-DeployZip -LocalZip $zipLocal -ZipName $zipName

        Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
        Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
        Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
        Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
        Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
        Invoke-Ec2Step "replace project directory" "sudo rm -rf $remotePath && sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
        Invoke-RemoteUnzip -ZipName $zipName -DestPath $remotePath
        Restore-OperatorFiles -Key $Key -RemotePath $remotePath
        Invoke-Ec2Step "require compose file" "test -f $remotePath/docker-compose.yml"
        Invoke-Ec2Step "docker compose build" "cd $remotePath && sudo COMPOSE_BAKE=false docker compose build"
        Invoke-Ec2Step "docker compose up -d" "cd $remotePath && sudo COMPOSE_BAKE=false docker compose up -d"

        $edgeProj = Get-ZEdgeProject
        if ($edgeProj -and $edgeProj.Config.proxyContainer) {
            Invoke-Ec2Step "reload edge nginx (flush DNS cache for new container IP)" "sudo docker exec $($edgeProj.Config.proxyContainer) nginx -s reload"
        }
        Invoke-Ec2Step "record deploy time; remove remote zip" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath -ZipName $zipName)

        Wait-VerifyStaticBuild -Key $Key -Proj $Proj -PreZipBuildState $preZipBuild
        Invoke-Ec2PostDeployCleanup -Label $Key

        $Elapsed    = (Get-Date) - $DeployStart
        $ElapsedStr = "{0:mm\:ss}" -f $Elapsed
        Write-Host "`n--- [Done] $($Proj.label) deployed! ---" -ForegroundColor Green
        Write-DeployLocation -Proj $Proj
        Write-Host "Change Note: $ChangeNote" -ForegroundColor Cyan
        Write-Host "Deploy Time: $ElapsedStr ($([math]::Round($Elapsed.TotalSeconds))s)" -ForegroundColor DarkGray
    }
    finally {
        if (Test-Path -LiteralPath $zipLocal) {
            try { Remove-Item -LiteralPath $zipLocal -Force -ErrorAction Stop } catch { }
        }
        Set-Location -LiteralPath $prevLoc
    }
}

function Invoke-NextDeploy {
    param([string]$Key, $Proj, [string]$ChangeNote)

    $DeployStart = Get-Date
    $prevLoc     = Get-Location
    $root        = $Proj.localRoot
    $remotePath  = $Proj.remote.path
    # Same resolution as the python handler. Must be computed HERE: PowerShell
    # function scope means the copy in Invoke-PythonDeploy is invisible from
    # this one, and an unset variable interpolates to an empty string — so
    # "cd  && docker compose down" quietly runs in the home directory.
    $composeDir  = if ($Proj.remote.composeDir) { $Proj.remote.composeDir } else { $remotePath }
    $appSvc      = if ($Proj.remote.appService) { $Proj.remote.appService } else { "web" }
    $zipName     = Get-DeployZipName -Key $Key -Proj $Proj
    $zipLocal    = Join-Path $TempRoot $zipName
    $preZipBuild = $null

    try {
        if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
        Set-Location -LiteralPath $root

        Write-Host "`n=== $($Proj.label) deploy (nextjs) ===" -ForegroundColor Cyan
        Write-Host "Local zip: $zipLocal" -ForegroundColor DarkGray

        # Stamp the build version from git before zipping, if the project says
        # how (deploy.stampCmd in zconfig). The image has no .git - it is in the
        # archive excludes - so the number has to be written on this side of the
        # zip.
        #
        # Written, zipped, then reverted: zdeploy refuses a dirty tree, so a
        # stamp that dirtied it every deploy would block the next one. The
        # committed file stays a fallback for local dev; the number that ships
        # is derived from the commit being deployed.
        $stampCmd     = if ($Proj.deploy -and $Proj.deploy.stampCmd) { $Proj.deploy.stampCmd } else { $null }
        $stampFile    = if ($Proj.deploy -and $Proj.deploy.stampFile) { $Proj.deploy.stampFile } else { "public/build-version.json" }
        $stampedLabel = $null
        if ($stampCmd) {
            Write-Host "`n--- [0] Stamping build version from git ---" -ForegroundColor Cyan
            $out = & cmd /c $stampCmd 2>&1
            if ($LASTEXITCODE -ne 0) { throw "Build stamp failed: $out" }
            $stampedLabel = ($out | Select-Object -Last 1).ToString().Trim()
            Write-Host "  $stampedLabel  (derived from the commit, not a counter)" -ForegroundColor Gray
        }

        $preZipBuild = Read-JsonBuildVersion -FilePath (Join-Path $root "public\build-version.json")
        if ($preZipBuild -and -not $stampedLabel) {
            Write-Host "  Pre-zip build label: $(Get-LabelFromBuildJsonObj $preZipBuild)" -ForegroundColor Gray
        }

        Write-Host "`n--- [1] Zipping project files ---" -ForegroundColor Cyan
        New-ProjectArchive -SourcePath $root -DestinationZip $zipLocal -TopLevelExclude (Get-ArchiveExcludes -Project $Proj)

        if ($stampCmd) {
            # Tree back to clean now the number is inside the archive.
            git -C $root checkout -- $stampFile 2>&1 | Out-Null
        }

        Invoke-Ec2PreflightCleanup -ExtraZipsToRemove @("$RemoteHome/$zipName")

        Write-Host "`n--- [2] Uploading zip ---" -ForegroundColor Cyan
        Send-DeployZip -LocalZip $zipLocal -ZipName $zipName

        Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
        Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
        Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
        Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
        Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
        Invoke-Ec2Step "replace project directory" "sudo rm -rf $remotePath && sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"
        Invoke-RemoteUnzip -ZipName $zipName -DestPath $remotePath
        Restore-OperatorFiles -Key $Key -RemotePath $remotePath

        Write-Host "`n--- [4] Docker compose rebuild ---" -ForegroundColor Cyan
        # composeDir, not remotePath: a project whose compose lives in a
        # subdirectory (deploy/, infra/, ...) otherwise runs these against
        # whatever docker-compose.yml happens to sit at the project root. That
        # is usually the LOCAL DEV compose, which ships in the same archive —
        # so the deploy recycled a dev database, found no app service to build,
        # exited 0, and left the previous image serving. Verification passed
        # because the untouched old container still answered. Two deploys in a
        # row silently shipped nothing.
        # Assert locally, not just on the server: `test -d $composeDir` with an
        # empty value becomes bare `test -d`, which is TRUE (one non-empty
        # argument), so the remote guard cannot catch this.
        if ([string]::IsNullOrWhiteSpace($composeDir)) { throw "composeDir resolved empty for '$Key' - compose would run in the wrong directory." }
        Invoke-Ec2Step "require compose file" "test -f $composeDir/docker-compose.yml || test -f $composeDir/docker-compose.yaml"
        # BUILD BEFORE DOWN. This used to run `down` first, which took the site
        # offline for the whole build - minutes for a Next.js app - and left it
        # offline if the build failed. That is not hypothetical: one deploy
        # stopped the stack, the build did not finish, and the site served 502
        # for 19 hours with no container at all. The
        # old image keeps serving while the new one builds, so a failed build is
        # now harmless and the outage is the seconds between down and up.
        #
        # Named service, like the python handler: a compose file that does not
        # define it fails here instead of succeeding with nothing to do.
        Invoke-Ec2Step "docker compose build $appSvc" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose build $appSvc"
        Invoke-Ec2Step "docker compose down" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose down"
        Invoke-Ec2Step "docker compose up -d" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose up -d"

        if ($Proj.db -and $Proj.db.user -and $Proj.db.name) {
            $waitDb = "cd $composeDir && for i in `$(seq 1 30); do sudo docker compose exec -T db pg_isready -U $($Proj.db.user) -d $($Proj.db.name) >/dev/null 2>&1 && break; sleep 2; done"
            Invoke-Ec2Step "wait for postgres ready" $waitDb
        }
        if ($Proj.migrations -eq "prisma") {
            Invoke-Ec2Step "apply prisma migrations" "cd $composeDir && sudo docker compose exec -T $appSvc npx prisma migrate deploy"
        }
        Invoke-Ec2Step "record deploy timestamp; remove remote zip" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath -ZipName $zipName)

        Write-Host "`n--- [5] Verifying deployment ---" -ForegroundColor Cyan
        # Same precedence as the python handler. Do NOT probe http://<ec2-ip>:<prod-port>/
        # here: a compose stack behind the edge proxy usually publishes to
        # 127.0.0.1 only, so that probe can never answer and the old "is the port
        # open in the security group?" warning sent you chasing a firewall rule
        # for an app that was already up. See issue #28.
        if ($Proj.verify -and $Proj.verify.port) {
            Test-DeployHealth -Key $Key -Proj $Proj -TimeoutSec 60 | Out-Null
        } elseif ($Proj.domain) {
            $headers = @{ 'Host' = $Proj.domain }
            $deadline = (Get-Date).AddSeconds(60)
            $verified = $false
            while ((Get-Date) -lt $deadline) {
                Start-Sleep -Seconds 4
                try {
                    $resp = Invoke-WebRequest -Uri "http://$EC2_IP/" -Headers $headers -TimeoutSec 8 -ErrorAction Stop -UseBasicParsing
                    if ($resp.StatusCode -lt 500) {
                        Write-Host "  PASS - app is responding at https://$($Proj.domain)/" -ForegroundColor Green
                        $verified = $true
                        break
                    }
                } catch {
                    Write-Host "  App not ready yet - waiting..." -ForegroundColor DarkGray
                }
            }
            if (-not $verified) {
                Write-Host "  WARNING: no response for https://$($Proj.domain)/ within 60s." -ForegroundColor Yellow
            }
        } else {
            Write-Host "  Deploy finished - NOT verified." -ForegroundColor Yellow
            Write-Host "  No 'domain' and no 'verify' block in zconfig.json for '$Key'." -ForegroundColor Yellow
        }
        if ($stampedLabel -or $preZipBuild) {
            # The label that actually went into the archive: the derived one
            # when the project stamps, otherwise the committed stamp.
            $expectedLabel = if ($stampedLabel) { $stampedLabel } else { Get-LabelFromBuildJsonObj $preZipBuild }
            Wait-VerifyApiBuild -Key $Key -Proj $Proj -ExpectedLabel $expectedLabel -TimeoutSec 60 | Out-Null
        } else {
            Write-Host "  (No public/build-version.json - version verification skipped. See 'Enabling deploy verification' in README.)" -ForegroundColor DarkYellow
        }

        Invoke-Ec2PostDeployCleanup -Label $Key

        $Elapsed    = (Get-Date) - $DeployStart
        $ElapsedStr = "{0:mm\:ss}" -f $Elapsed
        Write-Host "`n--- [Done] $($Proj.label) deployed! ---" -ForegroundColor Green
        Write-DeployLocation -Proj $Proj -Pad 12
        Write-Host "Change Note: $ChangeNote" -ForegroundColor Cyan
        Write-Host "Deploy Time: $ElapsedStr ($([math]::Round($Elapsed.TotalSeconds))s)" -ForegroundColor DarkGray
    }
    finally {
        if (Test-Path -LiteralPath $zipLocal) {
            try { Remove-Item -LiteralPath $zipLocal -Force -ErrorAction Stop } catch { }
        }
        Set-Location -LiteralPath $prevLoc
    }
}

function Invoke-EdgeDeploy {
    param([string]$Key, $Proj)

    $root       = $Proj.localRoot
    $remotePath = $Proj.remote.path
    $pc         = $Proj.proxyContainer

    Write-Host "`n=== $($Proj.label) deploy (edge nginx ingress) ===" -ForegroundColor Cyan
    if (-not (Test-Path -LiteralPath $root)) { throw "Edge root not found: $root" }
    foreach ($required in @('docker-compose.yml', 'nginx.conf')) {
        if (-not (Test-Path -LiteralPath (Join-Path $root $required))) { throw "Missing $root\$required" }
    }

    Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
    # -R: docker creates mount-point subdirs (vendor/, fonts/) root-owned when
    # they are missing at compose up; a non-recursive chown leaves those
    # unwritable and every scp into them fails.
    Invoke-Ec2Step "ensure edge dir" "sudo mkdir -p $remotePath && sudo chown -R ${Ec2User}:${Ec2User} $remotePath"

    # Ship every top-level file in the edge folder — nginx.conf, compose, css,
    # htpasswd, whatever the proxy serves.
    $files = @(Get-ChildItem -LiteralPath $root -File | Where-Object { $_.Name -ne 'nul' })
    foreach ($f in $files) {
        Write-Host "  >> uploading $($f.Name)" -ForegroundColor DarkCyan
        scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
        if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
    }

    # Content subdirectories the proxy serves (fonts/, vendor/, ...) ship too —
    # only server-side state stays put. Skipping them is how the self-hosted
    # Chart.js and fonts silently never reached prod (charts rendered blank).
    # .pytest_cache is a local test artifact, already gitignored; it has no
    # business on the proxy box and only adds noise to the upload log.
    $skipDirs = @('nginx-logs', '.git', '.pytest_cache')
    $dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
    foreach ($d in $dirs) {
        Write-Host "  >> uploading $($d.Name)/ (recursive)" -ForegroundColor DarkCyan
        scp -r @SCP_OPTS -i $PEM_KEY $d.FullName "${SSH_TARGET}:$remotePath/"
        if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
    }

    $certMount = if ($Proj.certsSource) { "-v $($Proj.certsSource):/etc/letsencrypt/:ro " } else { "" }
    Invoke-Ec2Step "validate new nginx.conf" "sudo docker run --rm -v $remotePath/nginx.conf:/etc/nginx/nginx.conf:ro ${certMount}nginx:1.27-alpine nginx -t -c /etc/nginx/nginx.conf"

    # A container from an older compose project may still hold the proxy name;
    # docker refuses a second create with the same name, so remove it first.
    $rmStale = if ($pc) { "; sudo docker rm -f $pc 2>/dev/null || true" } else { "" }
    Invoke-Ec2Step "edge: compose down + remove stale proxy" "cd $remotePath && sudo docker compose down 2>/dev/null || true$rmStale"
    Invoke-Ec2Step "edge compose up -d" "cd $remotePath && sudo docker compose up -d"
    if ($pc) {
        Invoke-Ec2Step "edge nginx reload" "sudo docker exec $pc nginx -s reload || true"
    }
    Invoke-Ec2Step "fix nginx-logs permissions (if present)" "if [ -d $remotePath/nginx-logs ]; then sudo chmod 777 $remotePath/nginx-logs; sudo chmod 666 $remotePath/nginx-logs/*.log 2>/dev/null || true; fi"
    Invoke-Ec2Step "record deploy stamp" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath)
    Write-Host "--- [Done] Edge proxy deploy finished ---" -ForegroundColor Green
}

function Invoke-StaticDeploy {
    param([string]$Key, $Proj)

    # Plain static sites - no build, no container of their own. The landing
    # container serves them straight off disk out of /srv/$host, so shipping
    # the files IS the deploy: there is nothing to restart afterwards.
    $root       = Join-Path $Proj.localRoot $Proj.siteDir
    $remotePath = $Proj.remote.path

    Write-Host "`n=== $($Proj.label) deploy (static files) ===" -ForegroundColor Cyan
    if (-not (Test-Path -LiteralPath $root)) { throw "Static site root not found: $root" }
    if (-not (Test-Path -LiteralPath (Join-Path $root 'index.html'))) { throw "Missing $root\index.html" }

    Invoke-Ec2Step "ensure site dir" "sudo mkdir -p $remotePath && sudo chown -R ${Ec2User}:${Ec2User} $remotePath"

    $files = @(Get-ChildItem -LiteralPath $root -File)
    foreach ($f in $files) {
        Write-Host "  >> uploading $($f.Name)" -ForegroundColor DarkCyan
        scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
        if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
    }

    # A large media file uploaded in place is served half-written to anyone who
    # requests it mid-copy. Ship the directory to a sibling, then swap it in.
    $skipDirs = @('.git', '.pytest_cache', 'node_modules')
    $dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
    foreach ($d in $dirs) {
        Write-Host "  >> uploading $($d.Name)/ (recursive, staged)" -ForegroundColor DarkCyan
        Invoke-Ec2Step "stage $($d.Name)" "rm -rf $remotePath/.staging-$($d.Name) && mkdir -p $remotePath/.staging-$($d.Name)"
        scp -r @SCP_OPTS -i $PEM_KEY "$($d.FullName)/*" "${SSH_TARGET}:$remotePath/.staging-$($d.Name)/"
        if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
        Invoke-Ec2Step "swap in $($d.Name)" "rm -rf $remotePath/$($d.Name) && mv $remotePath/.staging-$($d.Name) $remotePath/$($d.Name)"
    }

    Invoke-Ec2Step "record deploy stamp" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath)
    Write-Host "--- [Done] Static site deploy finished ---" -ForegroundColor Green
}

function Invoke-DockerDeploy {
    param([string]$Key, $Proj)

    $root       = $Proj.localRoot
    $remotePath = $Proj.remote.path

    Write-Host "`n=== $($Proj.label) deploy (docker compose) ===" -ForegroundColor Cyan
    if (-not (Test-Path -LiteralPath $root)) { throw "Project root not found: $root" }
    if (-not (Test-Path -LiteralPath (Join-Path $root "docker-compose.yml"))) { throw "Missing $root\docker-compose.yml" }

    Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
    Invoke-Ec2Step "ensure project dir" "sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath"

    $files = @(Get-ChildItem -LiteralPath $root -File -Force | Where-Object { $_.Name -ne 'nul' })
    foreach ($f in $files) {
        Write-Host "  >> uploading $($f.Name)" -ForegroundColor DarkCyan
        scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
        if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
    }

    # Config subdirectories ship too, exactly as the edge kind does. Uploading
    # top-level files ONLY was silently wrong for any stack that keeps config in
    # a directory: the compose file arrives, the containers restart, and the
    # config they read is whatever was already on the box. That is worse than a
    # failed deploy, because it reports success - a provisioning directory read
    # at container start (alert rules, datasources, mounted *.php) would never
    # reflect the change you just deployed.
    #
    # Skipped: $JunkDirNames (.git, __pycache__, .pytest_cache, ...) plus
    # anything the project lists in deploy.skipDirs. That list is how a stack
    # protects SERVER-SIDE STATE that happens to share the tree - a data/ holding
    # mailboxes or a time-series database must never be overwritten by whatever
    # the local checkout has (usually nothing, which is the dangerous case).
    # .github is CI config - it belongs in the repo and never on a deploy
    # target. It is not in $JunkDirNames because backups DO want it.
    $skipDirs = @($script:JunkDirNames) + @('.github')
    if ($Proj.deploy -and $Proj.deploy.skipDirs) { $skipDirs += @($Proj.deploy.skipDirs) }
    $dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
    foreach ($d in $dirs) {
        Write-Host "  >> uploading $($d.Name)/ (recursive)" -ForegroundColor DarkCyan
        scp -r @SCP_OPTS -i $PEM_KEY $d.FullName "${SSH_TARGET}:$remotePath/"
        if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
    }

    # Built here or pulled from a registry - see Get-DockerImageStep for why
    # a build-from-source stack cannot use `pull` and silently ships nothing.
    $imageStep = Get-DockerImageStep -Proj $Proj -RemotePath $remotePath
    Invoke-Ec2Step $imageStep.Label $imageStep.Command
    Invoke-Ec2Step "docker compose up -d" "cd $remotePath && sudo docker compose up -d"

    Invoke-Ec2PostDeployCleanup -Label $Key
    Invoke-Ec2Step "record deploy stamp" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath)
    Write-Host "`n--- [Done] $($Proj.label) deploy finished ---" -ForegroundColor Green
    Write-DeployLocation -Proj $Proj
}

# Pseudo-project "ztokens": not a zconfig entry, no compose stack. Runs
# `ztokens -Publish` from the sibling ztokens repo so the public zscripts page
# has current data. A failure here warns rather than aborting the rest of the
# deploy list - it's a nice-to-have refresh, not a deploy step.
function Invoke-ZTokensPublish {
    Write-Host "`n=== ztokens: refreshing live-usage stats ===" -ForegroundColor Cyan
    $ztokensScript = Join-Path (Split-Path -Parent $PSScriptRoot) "ztokens\ztokens.ps1"
    if (-not (Test-Path -LiteralPath $ztokensScript)) {
        Write-Host "  ztokens.ps1 not found at $ztokensScript - skipping." -ForegroundColor Yellow
        return
    }
    # Post-condition, not decoration. This step ran clean for five days while
    # publishing nothing: ztokens.ps1 had no -Publish switch, and as a simple
    # script PowerShell swallowed the unknown parameter into $args rather than
    # failing. The catch below never fired because nothing threw. So the check
    # is not "did it throw" but "did the file actually move".
    $statsFile = Join-Path (Split-Path -Parent $PSScriptRoot) "www\public\token-stats.json"
    $before = if (Test-Path -LiteralPath $statsFile) { (Get-Item -LiteralPath $statsFile).LastWriteTimeUtc } else { [datetime]::MinValue }
    try {
        & $ztokensScript -Publish
    } catch {
        Write-Host "  ztokens -Publish failed: $($_.Exception.Message)" -ForegroundColor Yellow
        return
    }
    $after = if (Test-Path -LiteralPath $statsFile) { (Get-Item -LiteralPath $statsFile).LastWriteTimeUtc } else { [datetime]::MinValue }
    if ($after -le $before) {
        Write-Host "  WARNING: token-stats.json was not rewritten - the site will ship the old numbers." -ForegroundColor Yellow
        if ($before -eq [datetime]::MinValue) {
            Write-Host "    $statsFile does not exist." -ForegroundColor DarkGray
        } else {
            Write-Host ("    Still dated {0:yyyy-MM-dd HH:mm} local." -f $before.ToLocalTime()) -ForegroundColor DarkGray
        }
        Write-Host "    Run 'ztokens -Publish' by hand to see why." -ForegroundColor DarkGray
    }
}

# ── Dispatch ─────────────────────────────────────────────────────────────────

foreach ($key in $Projects) {
    # 'ztokens' matches the tool it runs (ztokens.cmd / ztokens.ps1). The old
    # singular 'ztoken' still works so existing habits and any script that
    # already calls it keep running.
    if ($key -in @('ztokens', 'ztoken')) { Invoke-ZTokensPublish; continue }
    $proj = Get-ZProject -Key $key
    Invoke-DeployGitPull -Proj $proj   # no-op unless deploy.gitPull is set
    switch ([string]$proj.kind) {
        "python" { Invoke-PythonDeploy -Key $key -Proj $proj -ChangeNote $Note }
        "vite"   { Invoke-ViteDeploy   -Key $key -Proj $proj -ChangeNote $Note }
        "nextjs" { Invoke-NextDeploy   -Key $key -Proj $proj -ChangeNote $Note }
        "edge"   { Invoke-EdgeDeploy   -Key $key -Proj $proj }
        "docker" { Invoke-DockerDeploy -Key $key -Proj $proj }
        "static" { Invoke-StaticDeploy -Key $key -Proj $proj }
        default  { throw "No deploy handler for kind '$($proj.kind)' (project '$key'). Add an Invoke-<Kind>Deploy function in zdeploy.ps1." }
    }
}
# The timestamp goes through Stop-ZTracking as the FinalNote so it lands after
# the tracking footer and before the trailing blank lines - the last thing on
# screen, which is the point: scroll to the bottom and you can see how long ago
# this deployed. Only reached on success; a failed deploy throws out of the loop
# above, so this never claims a deploy that didn't happen.
Stop-ZTracking -FinalNote ("Last deployed at {0}" -f (Get-Date -Format "MM/dd/yyyy hh:mm:ss tt"))