diff --git a/CHANGELOG.md b/CHANGELOG.md index fa35f39..f2c2393 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,34 @@ Notable changes to the Evomedia.net Token Savers. ## Unreleased ### Fixed +- **Deploys can no longer hang forever on an ssh prompt** — every + deploy-path `ssh`/`scp` now carries `BatchMode=yes` plus connect and + keepalive timeouts (`Get-Ec2SshOpts` in `ZHelpers.ps1`). Without + `BatchMode`, ssh prompts for a passphrase or password and waits + indefinitely; because the deploy pipes stderr through the pipeline, the + prompt never reaches the screen and the run just stops under whatever + step label printed last, with no explanation. Now it fails immediately — + there is no prompt on this path worth answering. `ServerAlive*` bounds a + session that dies mid-command (dropped VPN, sleeping laptop, rebooting + host) to about a minute instead of hanging. +- **Vendored archives survive the archive filter** — files under a + `vendor/` directory are exempt from the "no archives in the zip" rule. + A project that vendors a dependency as `vendor/*.tgz` needs it in the + deploy zip; dropping it makes a Dockerfile's `COPY vendor ./vendor` + fail at image build, a confusing way to learn the filter ate a build + input. +- **`unzip` install is idempotent** — the remote step ran + `apt-get update && apt-get install -y unzip` on every deploy; it now + checks `command -v unzip` first and skips the apt round-trip when the + binary is already there. +- **`zdeploy` edge kind now ships asset subdirectories** (#42) — the edge + deploy uploaded top-level files only, so a project self-hosting assets + in folders (`fonts/`, `vendor/`) lost them on every deploy: docker + created empty root-owned mount points and nginx served 404s from them, + which shows up as fonts silently falling back and vendored JS never + loading. Every subdirectory except `nginx-logs/` and `.git/` now ships + recursively, and the `ensure edge dir` chown is recursive so scp into + docker-created root-owned dirs cannot fail. - **`zdeploy` no longer deletes operator-managed files on deploy** (#2) — the project-directory replacement preserved only `./.env`, silently destroying every other server-side file (`.env.db`, staged signing diff --git a/CHECKSUMS.txt b/CHECKSUMS.txt index fb64473..043ceab 100644 --- a/CHECKSUMS.txt +++ b/CHECKSUMS.txt @@ -8,14 +8,14 @@ c336a61cb8563736bd6550671d7dc90b14702a43f25a97b1ac15ffd64605fbf6 zbackup_ec2.cm 20e6282901a919aa32ea717c739ba2dc9657426dff860f0e5fce201edefd99da zchecksums.cmd 692b7b0ff9e51e5d9b880358b33c54d2db4f967297a5350e154df1ce6f06ebc6 zchecksums.ps1 b56085112e7c573926a70a58872163b71416d58ce91ea7114b1f5de5fc96c982 zdeploy.cmd -c4189cef72368487af4524a00603967d7c8dc03c58ace5d4dd1e264ce1d22bd9 zdeploy.ps1 +6005d581c1f86cfcaaf74b8b39205d247c0d9ed44c5f113aaa3282fd17161199 zdeploy.ps1 fa817d3bd7bba98b03b411d3c5a82b210d3cca316ac0c9785afa5b4ac5fe0766 zec2.cmd 5f70432af23df3459a98d9ad6a0da723783f2269dd5f5ab8ecdd6366ebc84942 zec2.ps1 4df94eda29b6f5b2cb6bfa63e0add4d90ea24d936305f36a69988e8dd5d69154 zec2_rotatekeys.cmd cc980bec4a9205a774c05ee1a7e0473e604adf5370b548015a5cb47412dd5853 zec2_rotatekeys.ps1 fae88c3d77efaef1a60d8d74bd0ca880ca67f85a2ab302fd182dfeb33e9465ce zec2online.cmd daf58f09cf118128c69282d51bbcece71cdeb1ce423737f6583ed00be460d98f zec2online.ps1 -aa2ef6f01e0fd1a478b79eaebcb9ea64bdbba33be91363b2fd3e65b58198d854 ZHelpers.ps1 +6a48be6f36f7f979c9fbdd996a6526f71978beb37913d3e45c74fecbe94098ee ZHelpers.ps1 bace82f5efc0cf63f260ccd5e134032029b70fe633900d3a6345f7eb079c121e zkill.cmd 443af456a5ababda04d882abf28f18079473708a90249c7294d03f8120db60f6 zkill.ps1 0e95bebabe8e56894c19519b38f57544bfb6903a3fe2f093fb7bf592c68b86da ZKiller.ps1 diff --git a/ZHelpers.ps1 b/ZHelpers.ps1 index cb9117c..f0e5c69 100644 --- a/ZHelpers.ps1 +++ b/ZHelpers.ps1 @@ -11,6 +11,13 @@ $script:ArchiveExtensions = @( '.bz2', '.xz', '.lz', '.lzma', '.cab', '.jar', '.war', '.ear', '.z', '.zst', '.zstd' ) +# Directories whose archives are BUILD INPUTS, not incidental bloat, and so are +# exempt from ArchiveExtensions. A project that vendors a dependency as +# vendor/*.tgz (common when a bundler cannot resolve `file:` links outside the +# project root) needs that tarball in the deploy zip — dropping it makes a +# Dockerfile's `COPY vendor ./vendor` fail at image build time, which is a +# confusing way to discover the archive filter ate a required file. +$script:ArchiveKeepDirNames = @('vendor') $script:ScriptExtensions = @('.ps1', '.cmd', '.bat') $script:JunkExtensions = @( '.swp', '.swo', '.swn', '.tmp', '.orig', '.rej', '.bak', @@ -115,6 +122,34 @@ function Get-Ec2Home { return "/home/$((Get-ZConfig).ec2.user)" } +# Options every deploy-path ssh/scp carries. Splat with @sshOpts. +# +# BatchMode=yes is the one that matters. Without it ssh PROMPTS - for a +# passphrase, a password, a sudo password - and waits forever. The deploy pipes +# stderr into the pipeline (2>&1 | ForEach-Object) so the prompt is swallowed +# on its way to the screen: the run simply stops under whatever step label was +# printed last, with nothing to explain it and no obvious reason why that +# particular step would be slow. One deploy appeared to hang on "ensure shared +# web network", a step whose entire body is `docker network create web +# 2>/dev/null || true` against a network that already existed. +# +# There is no prompt here you would ever want to answer - a deploy key is +# unencrypted and sudo on the box is passwordless - so failing immediately is +# strictly better than waiting on input that is never coming. +# +# ConnectTimeout bounds the TCP connect. ServerAlive* bound everything after +# it, so a session that dies mid-command (dropped VPN, laptop asleep, host +# rebooting) errors out in about a minute instead of hanging indefinitely. +function Get-Ec2SshOpts { + return @( + '-o', 'StrictHostKeyChecking=no', + '-o', 'BatchMode=yes', + '-o', 'ConnectTimeout=15', + '-o', 'ServerAliveInterval=15', + '-o', 'ServerAliveCountMax=4' + ) +} + # Run one bash command on the server; throw on non-zero exit. function Invoke-Ec2Step { param( @@ -132,7 +167,8 @@ function Invoke-Ec2Step { # to Continue locally (function-scoped, auto-reverts) and flatten stderr # into normal output, so only the actual exit status decides success. $ErrorActionPreference = 'Continue' - ssh -o StrictHostKeyChecking=no -i $cfg.ec2.pemKey (Get-Ec2Target) $Bash 2>&1 | + $sshOpts = Get-Ec2SshOpts + ssh @sshOpts -i $cfg.ec2.pemKey (Get-Ec2Target) $Bash 2>&1 | ForEach-Object { "$_" } if ($LASTEXITCODE -ne 0) { $msg = "Remote step failed: '$Label' (exit $LASTEXITCODE)." @@ -227,6 +263,18 @@ function Get-ArchiveExcludes { # ── Archive builder ────────────────────────────────────────────────────────── +# True when a file sits inside a directory named in $ArchiveKeepDirNames, i.e. +# its archive extension is a build input and must survive the archive filter. +function Test-InArchiveKeepDir { + param([Parameter(Mandatory)][System.IO.FileInfo] $File) + $dir = $File.DirectoryName + if (-not $dir) { return $false } + foreach ($segment in ($dir -split '[\\/]')) { + if ($script:ArchiveKeepDirNames -contains $segment) { return $true } + } + return $false +} + # Build a zip from a source directory (deploy/backup). # - $TopLevelExclude: skip these entries at the source root # - Archive/script/junk filters and $JunkDirNames pruning apply recursively @@ -313,7 +361,9 @@ function New-ProjectArchive { foreach ($f in $allFiles) { $ext = $f.Extension if ($ext) { $ext = $ext.ToLowerInvariant() } - if ($script:ArchiveExtensions -contains $ext) { $archivesSkipped++; continue } + if ($script:ArchiveExtensions -contains $ext -and -not (Test-InArchiveKeepDir $f)) { + $archivesSkipped++; continue + } if (-not $IncludeScriptFiles -and $script:ScriptExtensions -contains $ext) { $scriptsSkipped++; continue } if ($script:JunkExtensions -contains $ext) { $junkExtSkipped++; continue } if ($junkNameSet.Contains($f.Name)) { $junkNameSkipped++; continue } diff --git a/tests/NewProjectArchive.Tests.ps1 b/tests/NewProjectArchive.Tests.ps1 index 7088b3d..72ae781 100644 --- a/tests/NewProjectArchive.Tests.ps1 +++ b/tests/NewProjectArchive.Tests.ps1 @@ -211,6 +211,23 @@ Describe "New-ProjectArchive - file filters" { $e = Get-ArchivedEntries -Paths @("app.py", "styles.css", "index.html", "data.json") foreach ($f in @("app.py", "styles.css", "index.html", "data.json")) { $e | Should -Contain $f } } + + It "keeps an archive inside vendor/ - a vendored tarball is a build input" { + # Dockerfiles COPY vendor/ wholesale; dropping the tarball there fails + # the image build on the server, far from the filter that ate it. + $e = Get-ArchivedEntries -Paths @("app.py", "vendor/sdk-1.0.0.tgz") + $e | Should -Contain "vendor/sdk-1.0.0.tgz" + } + + It "keeps an archive in a nested vendor/ directory" { + $e = Get-ArchivedEntries -Paths @("app.py", "packages/api/vendor/sdk.tgz") + $e | Should -Contain "packages/api/vendor/sdk.tgz" + } + + It "still skips an archive outside vendor/ (the exemption is not global)" { + $e = Get-ArchivedEntries -Paths @("app.py", "assets/bundle.tgz") + $e | Should -Not -Contain "assets/bundle.tgz" + } } Describe "New-ProjectArchive - IncludeScriptFiles" { diff --git a/zdeploy.ps1 b/zdeploy.ps1 index ed135b6..0a26c14 100644 --- a/zdeploy.ps1 +++ b/zdeploy.ps1 @@ -52,6 +52,7 @@ $STACK_ROOT = $cfg.ec2.stackRoot $SSH_TARGET = Get-Ec2Target $RemoteHome = Get-Ec2Home $Ec2User = $cfg.ec2.user +$SSH_OPTS = Get-Ec2SshOpts # see ZHelpers.ps1 - these are what stop a deploy hanging $TempRoot = $cfg.paths.temp if (-not (Test-Path -LiteralPath $TempRoot)) { @@ -108,7 +109,7 @@ function Invoke-Ec2PreflightCleanup { "echo available_mb=`$avail_mb", "if [ `"`$avail_mb`" -lt 1500 ]; then echo 'ERROR: less than 1.5 GB free on /. Grow the root volume or run: sudo docker system prune -af' >&2; exit 11; fi" ) -join '; ' - ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET $preflightCmd + ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $preflightCmd if ($LASTEXITCODE -ne 0) { throw "Server pre-flight cleanup failed (exit $LASTEXITCODE). Root volume too full (need ~1.5 GB free, ideally 3+)." } @@ -126,7 +127,7 @@ function Invoke-Ec2PostDeployCleanup { "sudo find /var/lib/docker/containers/ -name '*-json.log' -size +50M -exec truncate -s 0 {} + 2>/dev/null || true", "df -h /" ) -join '; ' - ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET $cmd + ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $cmd if ($LASTEXITCODE -ne 0) { Write-Host " Post-deploy cleanup returned non-zero exit ($LASTEXITCODE); continuing." -ForegroundColor DarkYellow } @@ -134,7 +135,7 @@ function Invoke-Ec2PostDeployCleanup { function Send-DeployZip { param([string]$LocalZip, [string]$ZipName) - scp -i $PEM_KEY $LocalZip "${SSH_TARGET}:$RemoteHome/" + scp @SSH_OPTS -i $PEM_KEY $LocalZip "${SSH_TARGET}:$RemoteHome/" if ($LASTEXITCODE -ne 0) { throw "SCP upload failed (exit $LASTEXITCODE). Likely server disk space. Try: rm -f $RemoteHome/$ZipName" } @@ -196,7 +197,7 @@ function Wait-VerifyStaticBuild { if ($containerName) { # build-version.json may be blocked from external requests by the edge # proxy; read it inside the running container instead. - $raw = ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET ` + $raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET ` "sudo docker exec $containerName cat /usr/share/nginx/html/build-version.json 2>/dev/null" if ($raw) { $r = $raw | ConvertFrom-Json -ErrorAction Stop } } else { @@ -266,7 +267,7 @@ function Test-DeployHealth { # -L: an app whose "/" redirects (e.g. Next.js "/" -> "/login") answers a # 307 whose body is a few bytes or empty, which reads as "not ready". # Follow to the page that actually renders before judging. - $raw = ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET "curl -sL -m 8 http://localhost:$port$path" + $raw = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "curl -sL -m 8 http://localhost:$port$path" $body = ($raw | Out-String).Trim() if ($LASTEXITCODE -eq 0 -and $body) { if (-not $expect -or $body.Contains($expect)) { @@ -344,7 +345,7 @@ function Invoke-PythonDeploy { Send-DeployZip -LocalZip $zipLocal -ZipName $zipName Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan - Invoke-Ec2Step "apt-get install unzip" "sudo apt-get update -qq && sudo apt-get install -y unzip" + Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }" Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT" Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true" Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath @@ -360,7 +361,7 @@ function Invoke-PythonDeploy { Write-Host "`n--- [4] Incrementing build version ---" -ForegroundColor Cyan $BumpCmd = "cd $composeDir && sudo docker compose exec -T $appSvc python scripts/build_version_tool.py bump" for ($attempt = 1; $attempt -le 5; $attempt++) { - $output = ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET $BumpCmd + $output = ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET $BumpCmd if ($LASTEXITCODE -eq 0 -and $output) { $BuildVersion = ($output | Select-Object -Last 1).ToString().Trim() break @@ -371,7 +372,7 @@ function Invoke-PythonDeploy { if (-not $BuildVersion) { throw "Build version bump failed after 5 attempts" } python $versionTool set $BuildVersion | Out-Null - ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET "echo '$BuildVersion' | sudo tee $remotePath/.build_version > /dev/null" + ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "echo '$BuildVersion' | sudo tee $remotePath/.build_version > /dev/null" $changelogTool = Join-Path $root "scripts\build_changelog_tool.py" if (Test-Path -LiteralPath $changelogTool) { if ([string]::IsNullOrWhiteSpace($ChangeNote)) { $ChangeNote = "Build deployed" } @@ -379,7 +380,7 @@ function Invoke-PythonDeploy { } Write-Host "`n--- [5] Restarting app to pick up new version ---" -ForegroundColor Cyan - ssh -o StrictHostKeyChecking=no -i $PEM_KEY $SSH_TARGET "cd $composeDir && sudo COMPOSE_BAKE=false docker compose restart $appSvc" + ssh @SSH_OPTS -i $PEM_KEY $SSH_TARGET "cd $composeDir && sudo COMPOSE_BAKE=false docker compose restart $appSvc" if ($LASTEXITCODE -ne 0) { throw "App restart after build bump failed (exit $LASTEXITCODE)" } Wait-VerifyApiBuild -Key $Key -Proj $Proj -ExpectedLabel $BuildVersion -TimeoutSec 30 | Out-Null @@ -458,7 +459,7 @@ function Invoke-ViteDeploy { Send-DeployZip -LocalZip $zipLocal -ZipName $zipName Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan - Invoke-Ec2Step "apt-get install unzip" "sudo apt-get update -qq && sudo apt-get install -y unzip" + Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }" Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT" Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true" Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath @@ -526,7 +527,7 @@ function Invoke-NextDeploy { Send-DeployZip -LocalZip $zipLocal -ZipName $zipName Write-Host "`n--- [3] Unzipping and rebuilding on the server ---" -ForegroundColor Cyan - Invoke-Ec2Step "ensure unzip installed" "sudo apt-get update -qq && sudo apt-get install -y unzip" + Invoke-Ec2Step "ensure unzip installed" "command -v unzip >/dev/null 2>&1 || { sudo apt-get update -qq && sudo apt-get install -y unzip; }" Invoke-Ec2Step "ensure stack root" "sudo mkdir -p $STACK_ROOT && sudo chown ${Ec2User}:${Ec2User} $STACK_ROOT" Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true" Save-OperatorFiles -Key $Key -Proj $Proj -RemotePath $remotePath @@ -620,17 +621,32 @@ function Invoke-EdgeDeploy { } Invoke-Ec2Step "ensure shared web network" "sudo docker network create web 2>/dev/null || true" - Invoke-Ec2Step "ensure edge dir" "sudo mkdir -p $remotePath && sudo chown ${Ec2User}:${Ec2User} $remotePath" + # -R: docker creates mount-point subdirs (vendor/, fonts/) root-owned when + # they are missing at compose up; a non-recursive chown leaves those + # unwritable and every scp into them fails. + Invoke-Ec2Step "ensure edge dir" "sudo mkdir -p $remotePath && sudo chown -R ${Ec2User}:${Ec2User} $remotePath" # Ship every top-level file in the edge folder — nginx.conf, compose, css, - # htpasswd, whatever the proxy serves. Subdirectories (logs, certs) stay put. + # htpasswd, whatever the proxy serves. $files = @(Get-ChildItem -LiteralPath $root -File | Where-Object { $_.Name -ne 'nul' }) foreach ($f in $files) { Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan - scp -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/" + scp @SSH_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/" if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" } } + # Content subdirectories the proxy serves (fonts/, vendor/, ...) ship too — + # only server-side state stays put. Skipping them is how self-hosted assets + # silently never reach prod: docker creates empty mount-point dirs and nginx + # serves 404s from them, so fonts fall back and vendored JS never loads. + $skipDirs = @('nginx-logs', '.git') + $dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name }) + foreach ($d in $dirs) { + Write-Host " >> uploading $($d.Name)/ (recursive)" -ForegroundColor DarkCyan + scp -r @SSH_OPTS -i $PEM_KEY $d.FullName "${SSH_TARGET}:$remotePath/" + if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" } + } + $certMount = if ($Proj.certsSource) { "-v $($Proj.certsSource):/etc/letsencrypt/:ro " } else { "" } Invoke-Ec2Step "validate new nginx.conf" "sudo docker run --rm -v $remotePath/nginx.conf:/etc/nginx/nginx.conf:ro ${certMount}nginx:1.27-alpine nginx -t -c /etc/nginx/nginx.conf" @@ -662,7 +678,7 @@ function Invoke-DockerDeploy { $files = @(Get-ChildItem -LiteralPath $root -File -Force | Where-Object { $_.Name -ne 'nul' }) foreach ($f in $files) { Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan - scp -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/" + scp @SSH_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/" if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" } }