mirror of
https://github.com/kellymichels/zscripts-token-savers
synced 2026-10-07 07:18:18 +00:00
feat(zdeploy): add the static deploy kind, and a test that keeps this repo sanitized
Two things, both prompted by the same incident. STATIC KIND Plain static sites - no build, no container of their own; a shared web container serves them off disk, so shipping the files IS the deploy. Directories are staged to a sibling and swapped in with mv rather than copied in place, because a large media file uploaded in place is served half-written to anyone who requests it mid-copy. The swap is a rename, so the switch is atomic. Skips $JunkDirNames + .github + deploy.skipDirs, matching the docker kind rather than inventing a third convention. SANITIZATION TEST This repo is public and must stay standalone, but the toolkit is developed in a private checkout and copied here. Twice in three days a wholesale copy landed carrying real project names, internal hostnames, host disk figures, and references to scripts that exist only in the private copy. Both times a human reading the diff caught it - the control that fails exactly when a diff is 280 lines of good work with three bad words buried in it. So it is a test now. Seven rules: private project names, private product domains, private-only script names, local drive paths, the operator home path, the real key filename, and any IPv4 outside RFC 5737 documentation space and the private ranges. Two anti-vacuity guards, because a denylist that silently matches nothing is worse than no denylist: one asserts the file scan is non-empty, and one plants a known violation and requires the pattern to find it. The IP rule bounds on [\d.] rather than \d deliberately - this toolkit's own 5-segment version (v1.0.0.0.14) contains "0.0.0.14", which a plain digit boundary reads as an address. Verified against the real unsanitized copy that slipped through: 3 of the 7 rules fire, naming file and line. Tests: 231/231 (222 + 9 new). CHECKSUMS.txt refreshed.
This commit is contained in:
KellyMichels
parent
7b32bfada8
commit
3df54f4830
@ -8,7 +8,7 @@ dbd1b9c64fba16a308ffa8fa936f1bfda556b049aaa30ebe5400ca74b7e6aa5d zbackup_ec2.ps
|
||||
e03075f367a9ecee0f97438bc381044c74b7bd4e8cb4ff66b40048bba7ffd25a zchecksums.cmd
|
||||
8382ad5f972405678b73616f608a80e3eb1814c927ba104e446d36c4f9f5c66d zchecksums.ps1
|
||||
3d1064817ace57fe61c54104900775a0208fdf7e782043cced84b002347acb11 zdeploy.cmd
|
||||
c3b5eeac28cb4c17a8b74638466a7e8529af64dba79b11ad46b3d78c36b6b81f zdeploy.ps1
|
||||
5610abe7d4e979b3d8cdabb50710f23969454b647f0ab4f90d7cd68cc7f5ee38 zdeploy.ps1
|
||||
6fa05d7c47992801d0ad65095146764cc207b566dca85956b3152221ef9af368 zec2.cmd
|
||||
72217c04e8975f46b489bd7e223f9fda926d982a8e418fc2825c6aa4657f73b7 zec2.ps1
|
||||
d0702f372ec5e47e2632229c61b71a9184edf0775d6e23af74e3f919d13eadb9 zec2_rotatekeys.cmd
|
||||
|
||||
117
tests/Sanitization.Tests.ps1
Normal file
117
tests/Sanitization.Tests.ps1
Normal file
@ -0,0 +1,117 @@
|
||||
# Evomedia.net Token Savers — https://github.com/evomedia-net/evo.zscripts
|
||||
# Created by Kelly Michels · dev@evomedia.net
|
||||
# Licensed under the MIT License. See LICENSE.
|
||||
|
||||
# Sanitization.Tests.ps1 — this repo is public and must stay standalone.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# ---------------
|
||||
# The toolkit is developed in a private checkout and copied here. Twice in three
|
||||
# days a wholesale copy landed carrying environment-specific detail: real
|
||||
# project names, internal hostnames, host disk figures, and references to
|
||||
# scripts that exist only in the private copy. Each time it was caught by a
|
||||
# human reading the diff, which is exactly the control that fails when a diff is
|
||||
# 280 lines of good work with three bad words buried in it.
|
||||
#
|
||||
# So it is a test. A copy that reintroduces private detail turns the suite red
|
||||
# at the moment it happens rather than at review.
|
||||
#
|
||||
# WHAT IT CANNOT DO
|
||||
# -----------------
|
||||
# This is a denylist, so it proves the absence of KNOWN patterns, not the
|
||||
# absence of secrets. It is a regression net for a specific recurring mistake -
|
||||
# not a substitute for reading what you publish. Add a pattern whenever a new
|
||||
# private identifier appears; the cost of a stale entry is zero.
|
||||
|
||||
BeforeAll {
|
||||
$script:RepoRoot = Split-Path -Parent $PSScriptRoot
|
||||
|
||||
# Scanned: everything a reader of the published repo can see. Skipped:
|
||||
# .git (history is out of scope here), releases/ (published zips are
|
||||
# immutable by policy - rewriting one would break its checksum), and this
|
||||
# file, which necessarily contains every pattern it looks for.
|
||||
$script:Scanned = @(
|
||||
Get-ChildItem -LiteralPath $script:RepoRoot -Recurse -File |
|
||||
Where-Object {
|
||||
$_.Extension -in @('.ps1', '.cmd', '.md', '.txt', '.json', '.bats', '.sh') -and
|
||||
$_.FullName -notlike '*\.git\*' -and
|
||||
$_.FullName -notlike '*\releases\*' -and
|
||||
$_.Name -ne 'Sanitization.Tests.ps1'
|
||||
}
|
||||
)
|
||||
|
||||
# name -> what it is, so a failure explains itself
|
||||
# pattern -> regex, case-insensitive
|
||||
# Kept narrow on purpose: "evomedia.net" alone is legitimate here (the
|
||||
# attribution header and the repo URL), so only the drive path and specific
|
||||
# internal hosts are matched.
|
||||
$script:Denied = @(
|
||||
@{ Name = 'private project name'; Pattern = '\b(EvoCivilCode|EvoPlatform|DocketMail|SmartPlant\w*|ProvenSheet|evoehs|evoproven|evoaicc|evolocate|evoplatform)\b' }
|
||||
@{ Name = 'private product domain'; Pattern = '\b(smartplantehs\.com|provensheet\.com|evoehs\.com|civilcode\.evomedia\.net|dashboard\.evomedia\.net|webmail\.evomedia\.net|mail-admin\.evomedia\.net|docketmail\.evomedia\.net|cardiff\.evomedia\.net|platform\.evomedia\.net|ai\.evomedia\.net|git\.evomedia\.net|analytics\.evomedia\.net)\b' }
|
||||
@{ Name = 'private-only script'; Pattern = '\b(register_civilcode|register_docketmail|sp_seed_demo_prod|zpublish_stats|zcoverage|zmerge|zpull|zresume|swag_set_owner|provision_demo|apply_platform_config_fixes)\b' }
|
||||
@{ Name = 'local drive path'; Pattern = '[A-Za-z]:\\\\?evomedia\.net' }
|
||||
@{ Name = 'operator home path'; Pattern = '/home/ubuntu/' }
|
||||
@{ Name = 'real pem key name'; Pattern = 'evomedia-prod\.pem' }
|
||||
# RFC 5737 reserves 203.0.113.0/24 for documentation - that one is the
|
||||
# correct placeholder and must stay allowed, as are loopback and the
|
||||
# private ranges. Anything else that looks like a public IPv4 literal is
|
||||
# suspect.
|
||||
#
|
||||
# The boundaries are [\d.] rather than \d on purpose: this toolkit's own
|
||||
# 5-segment version (v1.0.0.0.14) contains "0.0.0.14", which a plain
|
||||
# digit boundary happily reads as an address. Refusing a match that
|
||||
# touches another dot rules out every version string without weakening
|
||||
# detection of a real address, which is always delimited by whitespace
|
||||
# or quotes.
|
||||
@{ Name = 'non-documentation IP'; Pattern = '(?<![\d.])(?!203\.0\.113\.)(?!127\.0\.0\.1)(?!0\.0\.0\.0)(?!255\.)(?!10\.)(?!192\.168\.)(?!172\.(1[6-9]|2\d|3[01])\.)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(?![\d.])' }
|
||||
)
|
||||
|
||||
function Get-Hits {
|
||||
param([string]$Pattern)
|
||||
$hits = @()
|
||||
foreach ($f in $script:Scanned) {
|
||||
$m = Select-String -LiteralPath $f.FullName -Pattern $Pattern -AllMatches -ErrorAction SilentlyContinue
|
||||
foreach ($line in $m) {
|
||||
$rel = $line.Path.Substring($script:RepoRoot.Length).TrimStart('\')
|
||||
$hits += "$rel`:$($line.LineNumber): $($line.Line.Trim())"
|
||||
}
|
||||
}
|
||||
return $hits
|
||||
}
|
||||
}
|
||||
|
||||
Describe "public repo carries no private detail" {
|
||||
|
||||
It "finds files to scan at all" {
|
||||
# Guards the guard: a bad filter here would make every test below pass
|
||||
# vacuously, which is worse than no test.
|
||||
$script:Scanned.Count | Should -BeGreaterThan 30
|
||||
}
|
||||
|
||||
It "contains no <Name>" -ForEach @(
|
||||
@{ Name = 'private project name' }
|
||||
@{ Name = 'private product domain' }
|
||||
@{ Name = 'private-only script' }
|
||||
@{ Name = 'local drive path' }
|
||||
@{ Name = 'operator home path' }
|
||||
@{ Name = 'real pem key name' }
|
||||
@{ Name = 'non-documentation IP' }
|
||||
) {
|
||||
$rule = $script:Denied | Where-Object { $_.Name -eq $Name }
|
||||
$hits = Get-Hits -Pattern $rule.Pattern
|
||||
$hits | Should -BeNullOrEmpty -Because "these look like private detail copied in from the internal toolkit:`n$($hits -join "`n")"
|
||||
}
|
||||
|
||||
It "still detects a planted violation" {
|
||||
# Mutation check. Without this the suite passes just as happily when the
|
||||
# patterns are broken as when the repo is clean - the failure mode that
|
||||
# makes a denylist worthless.
|
||||
$probe = Join-Path ([IO.Path]::GetTempPath()) ("sanitize-probe-" + [guid]::NewGuid().ToString("N") + ".ps1")
|
||||
try {
|
||||
Set-Content -LiteralPath $probe -Value '# deploy target /home/ubuntu/stack/example' -Encoding UTF8
|
||||
$found = Select-String -LiteralPath $probe -Pattern ($script:Denied | Where-Object { $_.Name -eq 'operator home path' }).Pattern
|
||||
$found | Should -Not -BeNullOrEmpty
|
||||
}
|
||||
finally { Remove-Item -LiteralPath $probe -Force -ErrorAction SilentlyContinue }
|
||||
}
|
||||
}
|
||||
43
zdeploy.ps1
43
zdeploy.ps1
@ -5,7 +5,7 @@
|
||||
|
||||
# zdeploy.ps1 — deploy any project defined in zconfig.json to the server.
|
||||
# Each project runs its own docker compose stack; the handler is picked by the
|
||||
# project's "kind": python | vite | nextjs | edge | docker.
|
||||
# project's "kind": python | vite | nextjs | edge | docker | static.
|
||||
#
|
||||
# Usage:
|
||||
# zdeploy <project> [<project> ...] [-Note "message"]
|
||||
@ -756,6 +756,46 @@ function Invoke-EdgeDeploy {
|
||||
Write-Host "--- [Done] Edge proxy deploy finished ---" -ForegroundColor Green
|
||||
}
|
||||
|
||||
function Invoke-StaticDeploy {
|
||||
param([string]$Key, $Proj)
|
||||
|
||||
# Plain static sites - no build, no container of their own. A shared web
|
||||
# container serves them straight off disk, so shipping the files IS the
|
||||
# deploy: there is nothing to restart afterwards.
|
||||
$root = Join-Path $Proj.localRoot $Proj.siteDir
|
||||
$remotePath = $Proj.remote.path
|
||||
|
||||
Write-Host "`n=== $($Proj.label) deploy (static files) ===" -ForegroundColor Cyan
|
||||
if (-not (Test-Path -LiteralPath $root)) { throw "Static site root not found: $root" }
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $root 'index.html'))) { throw "Missing $root\index.html" }
|
||||
|
||||
Invoke-Ec2Step "ensure site dir" "sudo mkdir -p $remotePath && sudo chown -R ${Ec2User}:${Ec2User} $remotePath"
|
||||
|
||||
$files = @(Get-ChildItem -LiteralPath $root -File)
|
||||
foreach ($f in $files) {
|
||||
Write-Host " >> uploading $($f.Name)" -ForegroundColor DarkCyan
|
||||
scp @SCP_OPTS -i $PEM_KEY $f.FullName "${SSH_TARGET}:$remotePath/"
|
||||
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($f.Name) (exit $LASTEXITCODE)" }
|
||||
}
|
||||
|
||||
# A large media file uploaded in place is served half-written to anyone who
|
||||
# requests it mid-copy. Ship each directory to a sibling, then swap it in -
|
||||
# the swap is a rename, so the switch is atomic and visitors never see a
|
||||
# partial file.
|
||||
$skipDirs = @($script:JunkDirNames) + @('.github')
|
||||
if ($Proj.deploy -and $Proj.deploy.skipDirs) { $skipDirs += @($Proj.deploy.skipDirs) }
|
||||
$dirs = @(Get-ChildItem -LiteralPath $root -Directory | Where-Object { $skipDirs -notcontains $_.Name })
|
||||
foreach ($d in $dirs) {
|
||||
Write-Host " >> uploading $($d.Name)/ (recursive, staged)" -ForegroundColor DarkCyan
|
||||
Invoke-Ec2Step "stage $($d.Name)" "rm -rf $remotePath/.staging-$($d.Name) && mkdir -p $remotePath/.staging-$($d.Name)"
|
||||
scp -r @SCP_OPTS -i $PEM_KEY "$($d.FullName)/*" "${SSH_TARGET}:$remotePath/.staging-$($d.Name)/"
|
||||
if ($LASTEXITCODE -ne 0) { throw "SCP failed for $($d.Name) (exit $LASTEXITCODE)" }
|
||||
Invoke-Ec2Step "swap in $($d.Name)" "rm -rf $remotePath/$($d.Name) && mv $remotePath/.staging-$($d.Name) $remotePath/$($d.Name)"
|
||||
}
|
||||
|
||||
Write-Host "--- [Done] Static site deploy finished ---" -ForegroundColor Green
|
||||
}
|
||||
|
||||
function Invoke-DockerDeploy {
|
||||
param([string]$Key, $Proj)
|
||||
|
||||
@ -860,6 +900,7 @@ foreach ($key in $Projects) {
|
||||
"nextjs" { Invoke-NextDeploy -Key $key -Proj $proj -ChangeNote $Note }
|
||||
"edge" { Invoke-EdgeDeploy -Key $key -Proj $proj }
|
||||
"docker" { Invoke-DockerDeploy -Key $key -Proj $proj }
|
||||
"static" { Invoke-StaticDeploy -Key $key -Proj $proj }
|
||||
default { throw "No deploy handler for kind '$($proj.kind)' (project '$key'). Add an Invoke-<Kind>Deploy function in zdeploy.ps1." }
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user