diff --git a/CHECKSUMS.txt b/CHECKSUMS.txt index ee763a4..8bbb104 100644 --- a/CHECKSUMS.txt +++ b/CHECKSUMS.txt @@ -8,14 +8,14 @@ cefe3201301d1bee8aa1cdd35818123836eda4d6975eb0f1d46352f45a484d35 zbackup_ec2.cm 7dae9f0f531cdd1a1ba46f9c52c41d26d2cf1168841c247d71dbea09362127b0 zchecksums.cmd 3d4bb2eee3aea0d8416b027a621e3e82b54c257a2cd694d4af65d00f828e1a3f zchecksums.ps1 6a5dd5f658338a44eae25379e51714550503cd0b85bb494a10af20138a0e948c zdeploy.cmd -e074a9340e13b8a01e4153062116c43a5e705f8f4b456fe4bca2fbb3f8a9ca3d zdeploy.ps1 +5d278d68eedd7cbc4e0243a0a33ecb65c3e71345313d5545c93bd3192c4c6399 zdeploy.ps1 c7bca9f557a816c2db5805fe4d0e90a4784976d7e2a177e3fa14b45465546702 zec2.cmd bb5b4d306ef2d4a4a02b77546052259ad6895b339741d8da112a5a2252e95ba1 zec2.ps1 73486d8df9a0afb5a87efb580036e71878c496c11f8074d12ccaad5ddc572393 zec2_rotatekeys.cmd 8cca7a1977bec02f569b78b8de1470307ffe62c4590443270704227671b4667a zec2_rotatekeys.ps1 f10e9d1ec91098e7e736bfbd57ad74d80a56b1121e34bdc381c0bfd2dd603007 zec2online.cmd 259b11058582b505a9cf8e0fd3aaff475ce4c131d40c8dfa00205090ad0a68c4 zec2online.ps1 -7451f625c730278907aa3eaac8ef18aa33b4aeec2cccb3c536dfeceb8232d396 ZHelpers.ps1 +54b1003368ffb8c321d4c5be433daefac473a449f2f6fa6718bfcd493c4dcfbe ZHelpers.ps1 4269c1850b05f2c13cfb0545e8e6f28429b6b7da7e986f766f1ba2f7bf2c9535 zkill.cmd 2c592ace4f565e9cedc9e296c42132115ea9ba8963f1718a6b85aafb76981702 zkill.ps1 e8f3c7207d68cf291a1e83a55dd8d49d71a68980830e5d4f6d89b07012cac314 ZKiller.ps1 diff --git a/zdeploy.ps1 b/zdeploy.ps1 index fbd174d..0415e82 100644 --- a/zdeploy.ps1 +++ b/zdeploy.ps1 @@ -9,6 +9,9 @@ # # Usage: # zdeploy [ ...] [-Note "message"] +# zdeploy -Scan # report what is merged but not shipped, then offer to deploy it +# zdeploy -s -Yes # same, unattended (no confirmation prompt) +# zdeploy -s ... # scan only these # zdeploy all # ztokens first, then every project (edge kinds next), stop at first failure # zdeploy ztokens # refresh the live-usage stats (see below) # @@ -43,7 +46,13 @@ param( [Parameter(Position = 0, ValueFromRemainingArguments = $true)] [string[]]$Projects = @(), - [string]$Note = "Build deployed" + [string]$Note = "Build deployed", + # -Scan / -s: report which projects have work on the default branch that is + # not live yet, then offer to deploy exactly those. See Get-DeployStatus. + [Alias('s')][switch]$Scan, + # Skip the confirmation prompt after a scan. Needed for unattended runs - + # Read-Host has no answer in a non-interactive shell and would throw. + [switch]$Yes ) $ErrorActionPreference = "Stop" @@ -65,12 +74,169 @@ if (-not (Test-Path -LiteralPath $TempRoot)) { New-Item -ItemType Directory -Path $TempRoot -Force | Out-Null } +# ── Scan: what is merged but not shipped ───────────────────────────────────── +# Answers "which projects have work on the default branch that is not live?" +# +# WHAT IT COMPARES +# ---------------- +# Every deploy leaves .last_deploy_sha and .last_deploy_utc in the project's +# remote path. The SHA is the real answer: deployed commit versus the current +# default-branch tip, exact regardless of clocks. +# +# .last_deploy_sha only exists from this change onward, so a project that has +# not been deployed since falls back to comparing the tip's COMMIT TIME against +# the deploy time. That is approximate on purpose and is labelled "~" in the +# output: commit time is when the work was authored, not when it merged, so a +# long-lived branch merged today carries an old timestamp and can read as +# already-shipped. The fallback disappears the first time each project deploys. +# +# WHAT IT DOES NOT DO +# ------------------- +# It does not judge whether the pending commits change anything shippable - a +# README-only commit still reads as pending. Deploying that is wasteful, not +# wrong, and the alternative (guessing which paths matter per project kind) is +# the sort of cleverness that eventually skips a real change. +function Get-DeployStatus { + param([string[]]$Keys) + + $cfgLocal = Get-ZConfig + $rows = @() + + # One ssh for every project rather than one each: this is a status read + # people will run often, and 15 round trips to answer one question is the + # difference between a habit and a chore. No $( ) and no embedded double + # quotes - see the note on Invoke-Ec2Step. + $parts = @() + foreach ($k in $Keys) { + $p = $cfgLocal.projects.$k + if (-not ($p -and $p.remote -and $p.remote.path)) { continue } + $rp = $p.remote.path + $parts += "printf '$k\t'; cat $rp/.last_deploy_sha 2>/dev/null | tr -d '\n'; printf '\t'; cat $rp/.last_deploy_utc 2>/dev/null | tr -d '\n'; printf '\n';" + } + $remote = @{} + if ($parts.Count -gt 0) { + $prev = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + # Deliberately not Invoke-Ec2Step: that prints a step header and throws + # on failure. A scan wants the output captured, and a box that cannot be + # reached should degrade to "unknown" rather than abort the report. + $sshOpts = Get-Ec2SshOpts + $lines = ssh @sshOpts -i $cfgLocal.ec2.pemKey (Get-Ec2Target) ($parts -join ' ') 2>&1 | + ForEach-Object { "$_" } + $ErrorActionPreference = $prev + foreach ($line in $lines) { + $f = $line -split "`t" + if ($f.Count -ge 3) { $remote[$f[0]] = @{ Sha = $f[1].Trim(); Utc = $f[2].Trim() } } + } + } + + foreach ($k in $Keys) { + $p = $cfgLocal.projects.$k + $row = [ordered]@{ Key = $k; Kind = $p.kind; State = ''; Detail = ''; Ahead = 0 } + $root = $p.localRoot + + # Not a test for .git in $root: a localRoot may point INTO a repo + # rather than at its top, when the deployable app is a subdirectory of + # the checkout. Testing for the folder reported every such project as + # having no checkout at all. Let git walk up instead. + $isRepo = $false + if ($root -and (Test-Path -LiteralPath $root)) { + $prev = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + git -C $root rev-parse --is-inside-work-tree 2>$null | Out-Null + $isRepo = ($LASTEXITCODE -eq 0) + $ErrorActionPreference = $prev + } + if (-not $isRepo) { + $row.State = 'no-repo'; $row.Detail = 'no git checkout' + $rows += [pscustomobject]$row; continue + } + + Push-Location -LiteralPath $root + try { + $prev = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + git fetch origin --prune --quiet + $default = (git symbolic-ref --short refs/remotes/origin/HEAD 2>$null) -replace '^origin/', '' + if (-not $default) { $default = 'main' } + $tip = (git rev-parse "origin/$default" 2>$null) + $tipUtc = (git show -s --format=%cI "origin/$default" 2>$null) + # Same exclusions as the deploy's own guard, or the scan would + # report a blocker zdeploy would happily run through: untracked + # files ship anyway, and build-version.json / CHANGELOG.md are + # written BY a deploy. + $dirty = git status --porcelain --untracked-files=no | Where-Object { + $name = ($_ -replace '^..\s+', '') -replace '^.*/', '' + @('build-version.json', 'CHANGELOG.md') -notcontains $name + } + $ErrorActionPreference = $prev + + if (-not $tip) { $row.State = 'no-repo'; $row.Detail = "no origin/$default"; $rows += [pscustomobject]$row; continue } + + $r = $remote[$k] + if (-not $r) { + $row.State = 'unknown'; $row.Detail = 'box unreachable' + } + elseif ($r.Sha) { + if ($r.Sha -eq $tip) { $row.State = 'current'; $row.Detail = $tip.Substring(0, 7) } + else { + $row.State = 'PENDING' + $n = (git rev-list --count "$($r.Sha)..origin/$default" 2>$null) + if (-not $n -or $LASTEXITCODE -ne 0) { $n = '?' } # deployed SHA not in this repo's history + $row.Ahead = $n + $row.Detail = "$n commit(s) since $($r.Sha.Substring(0, [Math]::Min(7, $r.Sha.Length)))" + } + } + elseif (-not $r.Utc) { + # NOT pending. No stamp means this project has never been + # deployed by a zdeploy that wrote one - which says nothing + # about whether it is behind. Calling it pending would have + # swept edge, the mail server and monitoring into an unattended + # run on no evidence at all, and edge in particular does not + # take a speculative deploy well. Deploy it once by name to set + # the baseline; every scan after that is exact. + $row.State = 'no-stamp'; $row.Detail = 'no deploy stamp - deploy once by name to baseline it' + } + else { + # Timestamp fallback - approximate, flagged with ~. + $deployedAt = [datetime]::MinValue + $ok = [datetime]::TryParse(($r.Utc -replace ' UTC$', ''), [ref]$deployedAt) + $tipAt = [datetime]::MinValue + $ok2 = [datetime]::TryParse($tipUtc, [ref]$tipAt) + if ($ok -and $ok2 -and $tipAt.ToUniversalTime() -gt $deployedAt) { + $row.State = 'PENDING' + $row.Ahead = '~' + $row.Detail = "~ tip $(($tipAt.ToUniversalTime()).ToString('MM-dd HH:mm')) > deploy $($r.Utc -replace ' UTC$','')" + } + else { + $row.State = 'current'; $row.Detail = "~ deployed $($r.Utc -replace ' UTC$','')" + } + } + + if ($dirty -and $row.State -eq 'PENDING') { + $row.State = 'BLOCKED' + $row.Detail = "$(@($dirty).Count) uncommitted file(s) - deploy would refuse" + } + } + finally { Pop-Location } + + $rows += [pscustomobject]$row + } + return $rows +} + +# A bare `zdeploy -s` means "look at everything", so it must not fall into the +# usage block below. +if ($Scan -and $Projects.Count -eq 0) { $Projects = @(Get-ZProjectKeys) } + if ($Projects.Count -eq 0) { $keys = (Get-ZProjectKeys) -join ', ' Write-Host "" Write-Host "Usage: zdeploy [ ...] | all | ztokens [-Note `"message`"]" -ForegroundColor Yellow Write-Host " Projects in zconfig.json: $keys" -ForegroundColor Gray Write-Host " 'all' deploys everything (edge kinds first) and stops at the first failure." -ForegroundColor Gray + Write-Host " -Scan / -s reports which projects have merged work that is not live, then offers to deploy just those." -ForegroundColor Gray + Write-Host " Add -Yes to skip the confirmation prompt. Edge still ships first." -ForegroundColor Gray Write-Host " 'ztokens' refreshes the live-usage stats published to the zscripts page." -ForegroundColor Gray Stop-ZTracking; exit 1 } @@ -90,6 +256,80 @@ if ($Projects -contains 'all') { # do the risky order, because this sort only ran for 'all'. # Order within each group is preserved, so an intentional sequence still holds # — notably `zdeploy ztokens evo`, where ztokens must still precede evo. +# Scan runs BEFORE the edge-first sort below, so whatever it selects still gets +# ordered by that rule - the proxy ships before the apps behind it, exactly as +# a hand-typed list would. +if ($Scan) { + Write-Host "`n=== zdeploy -Scan: what is merged but not shipped ===" -ForegroundColor Cyan + $status = Get-DeployStatus -Keys @($Projects | Where-Object { $_ -ne 'ztokens' }) + + Write-Host "" + foreach ($r in $status) { + $colour = switch ($r.State) { + 'PENDING' { 'Yellow' } + 'BLOCKED' { 'Red' } + 'no-stamp' { 'DarkYellow' } + 'current' { 'DarkGray' } + default { 'DarkYellow' } + } + Write-Host (" {0,-14} {1,-8} {2,-9} {3}" -f $r.Key, $r.Kind, $r.State, $r.Detail) -ForegroundColor $colour + } + + $pending = @($status | Where-Object { $_.State -eq 'PENDING' }) + $blocked = @($status | Where-Object { $_.State -eq 'BLOCKED' }) + $unknown = @($status | Where-Object { $_.State -eq 'unknown' }) + $nostamp = @($status | Where-Object { $_.State -eq 'no-stamp' }) + Write-Host "" + Write-Host (" {0} pending, {1} blocked, {2} no-stamp, {3} unknown, {4} current" -f ` + $pending.Count, $blocked.Count, $nostamp.Count, $unknown.Count, + @($status | Where-Object { $_.State -eq 'current' }).Count) -ForegroundColor Gray + + if ($blocked.Count -gt 0) { + Write-Host " Blocked projects are NOT deployed - commit or stash them, then re-run." -ForegroundColor Red + } + if ($nostamp.Count -gt 0) { + Write-Host " No-stamp projects are NOT selected - deploy each once by name to establish a baseline." -ForegroundColor DarkYellow + } + if ($unknown.Count -gt 0) { + # Silence here would read as "nothing to do", which is the one thing an + # unreachable box does not mean. + Write-Host " Unknown = the box did not answer for that project; its state is NOT 'current'." -ForegroundColor DarkYellow + } + + if ($pending.Count -eq 0) { + Write-Host "`n Nothing to deploy.`n" -ForegroundColor Green + Stop-ZTracking; exit 0 + } + + Write-Host "" + if (-not $Yes) { + # Two different ways a prompt can have nobody to answer it, and they + # fail differently: + # - a -NonInteractive host: Read-Host THROWS. Caught below. + # - redirected stdin (a pipe, a scheduled task, powershell.exe launched + # from another shell): Read-Host does NOT throw - it BLOCKS, waiting + # on a pipe that never answers. The first scan run this way sat for + # ten minutes with its table already printed but withheld behind the + # blocked pipeline. [Environment]::UserInteractive is $true in both + # cases, so it cannot be the test; IsInputRedirected can. + if ([Console]::IsInputRedirected) { + Write-Host " stdin is not a terminal - cannot prompt. Re-run with -Yes to deploy these $($pending.Count).`n" -ForegroundColor Yellow + Stop-ZTracking; exit 0 + } + $answer = $null + try { $answer = Read-Host " Deploy these $($pending.Count)? [y/N]" } + catch { + Write-Host " Non-interactive shell - cannot prompt. Re-run with -Yes to deploy these $($pending.Count).`n" -ForegroundColor Yellow + Stop-ZTracking; exit 0 + } + if ($answer -notmatch '^(y|yes)$') { + Write-Host " Aborted. Nothing deployed.`n" -ForegroundColor Yellow + Stop-ZTracking; exit 0 + } + } + $Projects = @($pending | ForEach-Object { $_.Key }) +} + $requested = @($Projects) $edgeKeys = @($Projects | Where-Object { $cfg.projects.$_.kind -eq 'edge' }) $restKeys = @($Projects | Where-Object { $cfg.projects.$_.kind -ne 'edge' }) @@ -101,6 +341,34 @@ if ($Projects.Count -gt 1) { Write-Host "Deploying: $($Projects -join ', ')$note" -ForegroundColor Cyan } +# The bash that stamps what just shipped. The timestamp has always been +# written; the SHA is what lets -Scan answer exactly rather than by clock +# comparison. Omitted rather than faked when the checkout is not a git repo - +# scan falls back to the timestamp, and a wrong SHA would be worse than none. +function Get-RecordDeployBash { + param( + [Parameter(Mandatory)]$Proj, + [Parameter(Mandatory)][string]$RemotePath, + # Optional: the edge, static and docker paths upload no zip, so there + # is nothing to remove - but they still ship, so they still stamp. + [string]$ZipName = '' + ) + $sha = '' + $root = $Proj.localRoot + if ($root -and (Test-Path -LiteralPath (Join-Path $root '.git'))) { + $prev = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + $sha = (git -C $root rev-parse HEAD 2>$null) + if ($LASTEXITCODE -ne 0) { $sha = '' } + $ErrorActionPreference = $prev + } + $cmd = "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $RemotePath/.last_deploy_utc > /dev/null" + # 40 hex characters, so it needs no quoting in the remote command. + if ($sha) { $cmd += " && printf '%s' $sha | sudo tee $RemotePath/.last_deploy_sha > /dev/null" } + if ($ZipName) { $cmd += " && rm -f $RemoteHome/$ZipName" } + return $cmd +} + function Get-DeployZipName { param([string]$Key, $Proj) if ($Proj.deploy -and $Proj.deploy.zipName) { return $Proj.deploy.zipName } @@ -507,7 +775,7 @@ function Invoke-PythonDeploy { Invoke-Ec2Step "require compose directory" "test -d $composeDir" Invoke-Ec2Step "docker compose build $appSvc" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose build $appSvc" Invoke-Ec2Step "docker compose up -d" "cd $composeDir && sudo COMPOSE_BAKE=false docker compose up -d" - Invoke-Ec2Step "record deploy time; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remotePath/.last_deploy_utc > /dev/null && rm -f $RemoteHome/$zipName" + Invoke-Ec2Step "record deploy time; remove remote zip" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath -ZipName $zipName) if ($hasVersionTool) { Write-Host "`n--- [4] Incrementing build version ---" -ForegroundColor Cyan @@ -651,7 +919,7 @@ function Invoke-ViteDeploy { if ($edgeProj -and $edgeProj.Config.proxyContainer) { Invoke-Ec2Step "reload edge nginx (flush DNS cache for new container IP)" "sudo docker exec $($edgeProj.Config.proxyContainer) nginx -s reload" } - Invoke-Ec2Step "record deploy time; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remotePath/.last_deploy_utc > /dev/null && rm -f $RemoteHome/$zipName" + Invoke-Ec2Step "record deploy time; remove remote zip" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath -ZipName $zipName) Wait-VerifyStaticBuild -Key $Key -Proj $Proj -PreZipBuildState $preZipBuild Invoke-Ec2PostDeployCleanup -Label $Key @@ -777,7 +1045,7 @@ function Invoke-NextDeploy { if ($Proj.migrations -eq "prisma") { Invoke-Ec2Step "apply prisma migrations" "cd $composeDir && sudo docker compose exec -T $appSvc npx prisma migrate deploy" } - Invoke-Ec2Step "record deploy timestamp; remove remote zip" "date -u +'%Y-%m-%d %H:%M:%S UTC' | sudo tee $remotePath/.last_deploy_utc > /dev/null && rm -f $RemoteHome/$zipName" + Invoke-Ec2Step "record deploy timestamp; remove remote zip" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath -ZipName $zipName) Write-Host "`n--- [5] Verifying deployment ---" -ForegroundColor Cyan # Same precedence as the python handler. Do NOT probe http://:/ @@ -890,6 +1158,7 @@ function Invoke-EdgeDeploy { Invoke-Ec2Step "edge nginx reload" "sudo docker exec $pc nginx -s reload || true" } Invoke-Ec2Step "fix nginx-logs permissions (if present)" "if [ -d $remotePath/nginx-logs ]; then sudo chmod 777 $remotePath/nginx-logs; sudo chmod 666 $remotePath/nginx-logs/*.log 2>/dev/null || true; fi" + Invoke-Ec2Step "record deploy stamp" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath) Write-Host "--- [Done] Edge proxy deploy finished ---" -ForegroundColor Green } @@ -927,6 +1196,7 @@ function Invoke-StaticDeploy { Invoke-Ec2Step "swap in $($d.Name)" "rm -rf $remotePath/$($d.Name) && mv $remotePath/.staging-$($d.Name) $remotePath/$($d.Name)" } + Invoke-Ec2Step "record deploy stamp" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath) Write-Host "--- [Done] Static site deploy finished ---" -ForegroundColor Green } @@ -978,6 +1248,7 @@ function Invoke-DockerDeploy { Invoke-Ec2Step "docker compose up -d" "cd $remotePath && sudo docker compose up -d" Invoke-Ec2PostDeployCleanup -Label $Key + Invoke-Ec2Step "record deploy stamp" (Get-RecordDeployBash -Proj $Proj -RemotePath $remotePath) Write-Host "`n--- [Done] $($Proj.label) deploy finished ---" -ForegroundColor Green Write-DeployLocation -Proj $Proj }