mirror of
https://github.com/kellymichels/zscripts-token-savers
synced 2026-10-07 07:18:18 +00:00
fix(zdeploy): stop deploys hanging on ssh prompts, keep vendored archives, make unzip install idempotent
- Get-Ec2SshOpts: every deploy-path ssh/scp now carries BatchMode=yes plus connect/keepalive timeouts. Without BatchMode ssh prompts and waits forever, and because the deploy pipes stderr through the pipeline the prompt never reaches the screen - the run just stops under the last step label with no explanation. - Archive filter exempts files under vendor/: a vendored *.tgz is a build input, and dropping it fails a Dockerfile COPY at image build. - unzip install checks command -v first instead of an apt round-trip on every deploy. Ported from the private script; three tests cover the vendor/ exemption.
This commit is contained in:
KellyMichels
parent
84764295b7
commit
4e2c832649
20
CHANGELOG.md
20
CHANGELOG.md
@ -11,6 +11,26 @@ Notable changes to the Evomedia.net Token Savers.
|
||||
## Unreleased
|
||||
|
||||
### Fixed
|
||||
- **Deploys can no longer hang forever on an ssh prompt** — every
|
||||
deploy-path `ssh`/`scp` now carries `BatchMode=yes` plus connect and
|
||||
keepalive timeouts (`Get-Ec2SshOpts` in `ZHelpers.ps1`). Without
|
||||
`BatchMode`, ssh prompts for a passphrase or password and waits
|
||||
indefinitely; because the deploy pipes stderr through the pipeline, the
|
||||
prompt never reaches the screen and the run just stops under whatever
|
||||
step label printed last, with no explanation. Now it fails immediately —
|
||||
there is no prompt on this path worth answering. `ServerAlive*` bounds a
|
||||
session that dies mid-command (dropped VPN, sleeping laptop, rebooting
|
||||
host) to about a minute instead of hanging.
|
||||
- **Vendored archives survive the archive filter** — files under a
|
||||
`vendor/` directory are exempt from the "no archives in the zip" rule.
|
||||
A project that vendors a dependency as `vendor/*.tgz` needs it in the
|
||||
deploy zip; dropping it makes a Dockerfile's `COPY vendor ./vendor`
|
||||
fail at image build, a confusing way to learn the filter ate a build
|
||||
input.
|
||||
- **`unzip` install is idempotent** — the remote step ran
|
||||
`apt-get update && apt-get install -y unzip` on every deploy; it now
|
||||
checks `command -v unzip` first and skips the apt round-trip when the
|
||||
binary is already there.
|
||||
- **`zdeploy` edge kind now ships asset subdirectories** (#42) — the edge
|
||||
deploy uploaded top-level files only, so a project self-hosting assets
|
||||
in folders (`fonts/`, `vendor/`) lost them on every deploy: docker
|
||||
|
||||
@ -8,14 +8,14 @@ c336a61cb8563736bd6550671d7dc90b14702a43f25a97b1ac15ffd64605fbf6 zbackup_ec2.cm
|
||||
20e6282901a919aa32ea717c739ba2dc9657426dff860f0e5fce201edefd99da zchecksums.cmd
|
||||
692b7b0ff9e51e5d9b880358b33c54d2db4f967297a5350e154df1ce6f06ebc6 zchecksums.ps1
|
||||
b56085112e7c573926a70a58872163b71416d58ce91ea7114b1f5de5fc96c982 zdeploy.cmd
|
||||
c847a5b8e679014e1d363c23966fcaf95e443e10cde4f76f197b27a0aa8b9746 zdeploy.ps1
|
||||
6005d581c1f86cfcaaf74b8b39205d247c0d9ed44c5f113aaa3282fd17161199 zdeploy.ps1
|
||||
fa817d3bd7bba98b03b411d3c5a82b210d3cca316ac0c9785afa5b4ac5fe0766 zec2.cmd
|
||||
5f70432af23df3459a98d9ad6a0da723783f2269dd5f5ab8ecdd6366ebc84942 zec2.ps1
|
||||
4df94eda29b6f5b2cb6bfa63e0add4d90ea24d936305f36a69988e8dd5d69154 zec2_rotatekeys.cmd
|
||||
cc980bec4a9205a774c05ee1a7e0473e604adf5370b548015a5cb47412dd5853 zec2_rotatekeys.ps1
|
||||
fae88c3d77efaef1a60d8d74bd0ca880ca67f85a2ab302fd182dfeb33e9465ce zec2online.cmd
|
||||
daf58f09cf118128c69282d51bbcece71cdeb1ce423737f6583ed00be460d98f zec2online.ps1
|
||||
aa2ef6f01e0fd1a478b79eaebcb9ea64bdbba33be91363b2fd3e65b58198d854 ZHelpers.ps1
|
||||
6a48be6f36f7f979c9fbdd996a6526f71978beb37913d3e45c74fecbe94098ee ZHelpers.ps1
|
||||
bace82f5efc0cf63f260ccd5e134032029b70fe633900d3a6345f7eb079c121e zkill.cmd
|
||||
443af456a5ababda04d882abf28f18079473708a90249c7294d03f8120db60f6 zkill.ps1
|
||||
0e95bebabe8e56894c19519b38f57544bfb6903a3fe2f093fb7bf592c68b86da ZKiller.ps1
|
||||
|
||||
54
ZHelpers.ps1
54
ZHelpers.ps1
@ -11,6 +11,13 @@ $script:ArchiveExtensions = @(
|
||||
'.bz2', '.xz', '.lz', '.lzma', '.cab', '.jar', '.war', '.ear', '.z',
|
||||
'.zst', '.zstd'
|
||||
)
|
||||
# Directories whose archives are BUILD INPUTS, not incidental bloat, and so are
|
||||
# exempt from ArchiveExtensions. A project that vendors a dependency as
|
||||
# vendor/*.tgz (common when a bundler cannot resolve `file:` links outside the
|
||||
# project root) needs that tarball in the deploy zip — dropping it makes a
|
||||
# Dockerfile's `COPY vendor ./vendor` fail at image build time, which is a
|
||||
# confusing way to discover the archive filter ate a required file.
|
||||
$script:ArchiveKeepDirNames = @('vendor')
|
||||
$script:ScriptExtensions = @('.ps1', '.cmd', '.bat')
|
||||
$script:JunkExtensions = @(
|
||||
'.swp', '.swo', '.swn', '.tmp', '.orig', '.rej', '.bak',
|
||||
@ -115,6 +122,34 @@ function Get-Ec2Home {
|
||||
return "/home/$((Get-ZConfig).ec2.user)"
|
||||
}
|
||||
|
||||
# Options every deploy-path ssh/scp carries. Splat with @sshOpts.
|
||||
#
|
||||
# BatchMode=yes is the one that matters. Without it ssh PROMPTS - for a
|
||||
# passphrase, a password, a sudo password - and waits forever. The deploy pipes
|
||||
# stderr into the pipeline (2>&1 | ForEach-Object) so the prompt is swallowed
|
||||
# on its way to the screen: the run simply stops under whatever step label was
|
||||
# printed last, with nothing to explain it and no obvious reason why that
|
||||
# particular step would be slow. One deploy appeared to hang on "ensure shared
|
||||
# web network", a step whose entire body is `docker network create web
|
||||
# 2>/dev/null || true` against a network that already existed.
|
||||
#
|
||||
# There is no prompt here you would ever want to answer - a deploy key is
|
||||
# unencrypted and sudo on the box is passwordless - so failing immediately is
|
||||
# strictly better than waiting on input that is never coming.
|
||||
#
|
||||
# ConnectTimeout bounds the TCP connect. ServerAlive* bound everything after
|
||||
# it, so a session that dies mid-command (dropped VPN, laptop asleep, host
|
||||
# rebooting) errors out in about a minute instead of hanging indefinitely.
|
||||
function Get-Ec2SshOpts {
|
||||
return @(
|
||||
'-o', 'StrictHostKeyChecking=no',
|
||||
'-o', 'BatchMode=yes',
|
||||
'-o', 'ConnectTimeout=15',
|
||||
'-o', 'ServerAliveInterval=15',
|
||||
'-o', 'ServerAliveCountMax=4'
|
||||
)
|
||||
}
|
||||
|
||||
# Run one bash command on the server; throw on non-zero exit.
|
||||
function Invoke-Ec2Step {
|
||||
param(
|
||||
@ -132,7 +167,8 @@ function Invoke-Ec2Step {
|
||||
# to Continue locally (function-scoped, auto-reverts) and flatten stderr
|
||||
# into normal output, so only the actual exit status decides success.
|
||||
$ErrorActionPreference = 'Continue'
|
||||
ssh -o StrictHostKeyChecking=no -i $cfg.ec2.pemKey (Get-Ec2Target) $Bash 2>&1 |
|
||||
$sshOpts = Get-Ec2SshOpts
|
||||
ssh @sshOpts -i $cfg.ec2.pemKey (Get-Ec2Target) $Bash 2>&1 |
|
||||
ForEach-Object { "$_" }
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
$msg = "Remote step failed: '$Label' (exit $LASTEXITCODE)."
|
||||
@ -227,6 +263,18 @@ function Get-ArchiveExcludes {
|
||||
|
||||
# ── Archive builder ──────────────────────────────────────────────────────────
|
||||
|
||||
# True when a file sits inside a directory named in $ArchiveKeepDirNames, i.e.
|
||||
# its archive extension is a build input and must survive the archive filter.
|
||||
function Test-InArchiveKeepDir {
|
||||
param([Parameter(Mandatory)][System.IO.FileInfo] $File)
|
||||
$dir = $File.DirectoryName
|
||||
if (-not $dir) { return $false }
|
||||
foreach ($segment in ($dir -split '[\\/]')) {
|
||||
if ($script:ArchiveKeepDirNames -contains $segment) { return $true }
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
# Build a zip from a source directory (deploy/backup).
|
||||
# - $TopLevelExclude: skip these entries at the source root
|
||||
# - Archive/script/junk filters and $JunkDirNames pruning apply recursively
|
||||
@ -313,7 +361,9 @@ function New-ProjectArchive {
|
||||
foreach ($f in $allFiles) {
|
||||
$ext = $f.Extension
|
||||
if ($ext) { $ext = $ext.ToLowerInvariant() }
|
||||
if ($script:ArchiveExtensions -contains $ext) { $archivesSkipped++; continue }
|
||||
if ($script:ArchiveExtensions -contains $ext -and -not (Test-InArchiveKeepDir $f)) {
|
||||
$archivesSkipped++; continue
|
||||
}
|
||||
if (-not $IncludeScriptFiles -and $script:ScriptExtensions -contains $ext) { $scriptsSkipped++; continue }
|
||||
if ($script:JunkExtensions -contains $ext) { $junkExtSkipped++; continue }
|
||||
if ($junkNameSet.Contains($f.Name)) { $junkNameSkipped++; continue }
|
||||
|
||||
@ -211,6 +211,23 @@ Describe "New-ProjectArchive - file filters" {
|
||||
$e = Get-ArchivedEntries -Paths @("app.py", "styles.css", "index.html", "data.json")
|
||||
foreach ($f in @("app.py", "styles.css", "index.html", "data.json")) { $e | Should -Contain $f }
|
||||
}
|
||||
|
||||
It "keeps an archive inside vendor/ - a vendored tarball is a build input" {
|
||||
# Dockerfiles COPY vendor/ wholesale; dropping the tarball there fails
|
||||
# the image build on the server, far from the filter that ate it.
|
||||
$e = Get-ArchivedEntries -Paths @("app.py", "vendor/sdk-1.0.0.tgz")
|
||||
$e | Should -Contain "vendor/sdk-1.0.0.tgz"
|
||||
}
|
||||
|
||||
It "keeps an archive in a nested vendor/ directory" {
|
||||
$e = Get-ArchivedEntries -Paths @("app.py", "packages/api/vendor/sdk.tgz")
|
||||
$e | Should -Contain "packages/api/vendor/sdk.tgz"
|
||||
}
|
||||
|
||||
It "still skips an archive outside vendor/ (the exemption is not global)" {
|
||||
$e = Get-ArchivedEntries -Paths @("app.py", "assets/bundle.tgz")
|
||||
$e | Should -Not -Contain "assets/bundle.tgz"
|
||||
}
|
||||
}
|
||||
|
||||
Describe "New-ProjectArchive - IncludeScriptFiles" {
|
||||
|
||||
29
zdeploy.ps1
29
zdeploy.ps1
@ -52,6 +52,7 @@ $STACK_ROOT = $cfg.ec2.stackRoot
|
||||
$SSH_TARGET = Get-Ec2Target
|
||||
$RemoteHome = Get-Ec2Home
|
||||
$Ec2User = $cfg.ec2.user
|
||||
$SSH_OPTS = Get-Ec2SshOpts # see ZHelpers.ps1 - these are what stop a deploy hanging
|
||||
|
||||
$TempRoot = $cfg.paths.temp
|
||||
if (-not (Test-Path -LiteralPath $TempRoot)) {
|
||||
@ -108,7 +109,7 @@ function Invoke-Ec2PreflightCleanup {
|
||||
"echo available_mb=`$avail_mb",
|
||||
"if [ `"`$avail_mb`" -lt 1500 ]; then echo 'ERROR: less than 1.5 GB free on /. Grow the root volume or run: sudo docker system prune -af' >&2; exit 11; fi"
|
||||
) -join '; '
|
||||
ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET $preflightCmd
|
||||
ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $preflightCmd
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Server pre-flight cleanup failed (exit $LASTEXITCODE). Root volume too full (need ~1.5 GB free, ideally 3+)."
|
||||
}
|
||||
@ -126,7 +127,7 @@ function Invoke-Ec2PostDeployCleanup {
|
||||
"sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true",
|
||||
"df -h /"
|
||||
) -join '; '
|
||||
ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET $cmd
|
||||
ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $cmd
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host " Post-deploy cleanup returned non-zero exit ($LASTEXITCODE); continuing." -ForegroundColor DarkYellow
|
||||
}
|
||||
@ -134,7 +135,7 @@ function Invoke-Ec2PostDeployCleanup {
|
||||
|
||||
function Send-DeployZip {
|
||||
param([string]$LocalZip, [string]$ZipName)
|
||||
scp -i $PEM_KEY $LocalZip "${SSH_TARGET}:$RemoteHome/"
|
||||
scp @SSH_OPTS -i $PEM_KEY $LocalZip "${SSH_TARGET}:$RemoteHome/"
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "SCP upload failed (exit $LASTEXITCODE). Likely server disk space. Try: rm -f $RemoteHome/$ZipName"
|
||||
}
|
||||
@ -196,7 +197,7 @@ function Wait-VerifyStaticBuild {
|
||||
if ($containerName) {
|
||||
# build-version.json may be blocked from external requests by the edge
|
||||
# proxy; read it inside the running container instead.
|
||||
$raw = ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET `
|
||||
$raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET `
|
||||
"sudo docker exec $containerName cat /usr/share/nginx/html/build-version.json 2>/dev/null"
|
||||
if ($raw) { $r = $raw | ConvertFrom-Json -ErrorAction Stop }
|
||||
} else {
|
||||
@ -266,7 +267,7 @@ function Test-DeployHealth {
|
||||
# -L: an app whose "/" redirects (e.g. Next.js "/" -> "/login") answers a
|
||||
# 307 whose body is a few bytes or empty, which reads as "not ready".
|
||||
# Follow to the page that actually renders before judging.
|
||||
$raw = ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET "curl -sL -m 8 http://localhost:$port$path"
|
||||
$raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "curl -sL -m 8 http://localhost:$port$path"
|
||||
$body = ($raw | Out-String).Trim()
|
||||
if ($LASTEXITCODE -eq 0 -and $body) {
|
||||
if (-not $expect -or $body.Contains($expect)) {
|
||||
@ -344,7 +345,7 @@ function Invoke-PythonDeploy {
|
||||
Send-DeployZip -LocalZip $zipLocal -ZipName $zipName
|
||||
|
||||
Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
|
||||
Invoke-Ec2Step "apt-get install unzip" "sudo apt-get update -qq && sudo apt-get install -y unzip"
|
||||
Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
|
||||
Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
|
||||
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
||||
Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
|
||||
@ -360,7 +361,7 @@ function Invoke-PythonDeploy {
|
||||
Write-Host "`n--- [4] Incrementing build version ---" -ForegroundColor Cyan
|
||||
$BumpCmd = "cd $composeDir && sudo docker compose exec -T $appSvc python scripts/build_version_tool.py bump"
|
||||
for ($attempt = 1; $attempt -le 5; $attempt++) {
|
||||
$output = ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET $BumpCmd
|
||||
$output = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $BumpCmd
|
||||
if ($LASTEXITCODE -eq 0 -and $output) {
|
||||
$BuildVersion = ($output | Select-Object -Last 1).ToString().Trim()
|
||||
break
|
||||
@ -371,7 +372,7 @@ function Invoke-PythonDeploy {
|
||||
if (-not $BuildVersion) { throw "Build version bump failed after 5 attempts" }
|
||||
|
||||
python $versionTool set $BuildVersion | Out-Null
|
||||
ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET "echo '$BuildVersion' | sudo tee $remotePath/.build_version > /dev/null"
|
||||
ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "echo '$BuildVersion' | sudo tee $remotePath/.build_version > /dev/null"
|
||||
$changelogTool = Join-Path $root "scripts\build_changelog_tool.py"
|
||||
if (Test-Path -LiteralPath $changelogTool) {
|
||||
if ([string]::IsNullOrWhiteSpace($ChangeNote)) { $ChangeNote = "Build deployed" }
|
||||
@ -379,7 +380,7 @@ function Invoke-PythonDeploy {
|
||||
}
|
||||
|
||||
Write-Host "`n--- [5] Restarting app to pick up new version ---" -ForegroundColor Cyan
|
||||
ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET "cd $composeDir && sudo COMPOSE_BAKE=false docker compose restart $appSvc"
|
||||
ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "cd $composeDir && sudo COMPOSE_BAKE=false docker compose restart $appSvc"
|
||||
if ($LASTEXITCODE -ne 0) { throw "App restart after build bump failed (exit $LASTEXITCODE)" }
|
||||
|
||||
Wait-VerifyApiBuild -Key $Key -Proj $Proj -ExpectedLabel $BuildVersion -TimeoutSec 30 | Out-Null
|
||||
@ -458,7 +459,7 @@ function Invoke-ViteDeploy {
|
||||
Send-DeployZip -LocalZip $zipLocal -ZipName $zipName
|
||||
|
||||
Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
|
||||
Invoke-Ec2Step "apt-get install unzip" "sudo apt-get update -qq && sudo apt-get install -y unzip"
|
||||
Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
|
||||
Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
|
||||
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
||||
Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
|
||||
@ -526,7 +527,7 @@ function Invoke-NextDeploy {
|
||||
Send-DeployZip -LocalZip $zipLocal -ZipName $zipName
|
||||
|
||||
Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan
|
||||
Invoke-Ec2Step "ensure unzip installed" "sudo apt-get update -qq && sudo apt-get install -y unzip"
|
||||
Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }"
|
||||
Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT"
|
||||
Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true"
|
||||
Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath
|
||||
@ -630,7 +631,7 @@ function Invoke-EdgeDeploy {
|
||||
$files = @(Get-ChildItem -LiteralPath $root -File | Where-Object { $_.Name -ne 'nul' })
|
||||
foreach ($f in $files) {
|
||||
Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan
|
||||
scp -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
||||
scp @SSH_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
||||
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
|
||||
}
|
||||
|
||||
@ -642,7 +643,7 @@ function Invoke-EdgeDeploy {
|
||||
$dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
|
||||
foreach ($d in $dirs) {
|
||||
Write-Host " >> uploading $($d.Name)/ (recursive)" -ForegroundColor DarkCyan
|
||||
scp -r -i $PEM_KEY $d.FullName "${SSH_TARGET}:$remotePath/"
|
||||
scp -r @SSH_OPTS -i $PEM_KEY $d.FullName "${SSH_TARGET}:$remotePath/"
|
||||
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
|
||||
}
|
||||
|
||||
@ -677,7 +678,7 @@ function Invoke-DockerDeploy {
|
||||
$files = @(Get-ChildItem -LiteralPath $root -File -Force | Where-Object { $_.Name -ne 'nul' })
|
||||
foreach ($f in $files) {
|
||||
Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan
|
||||
scp -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
||||
scp @SSH_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
||||
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user