974 lines
42 KiB
PowerShell
974 lines
42 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[string]$PgRoot = 'D:\pgsql17',
|
|
[string]$RuntimeRoot = (Join-Path ([IO.Path]::GetTempPath()) 'yovision-t014'),
|
|
[ValidateRange(1, 1440)]
|
|
[int]$ObservationMinutes = 30,
|
|
[string]$OutputPath = '',
|
|
[switch]$PreflightOnly
|
|
)
|
|
|
|
$ErrorActionPreference = 'Stop'
|
|
$ProgressPreference = 'SilentlyContinue'
|
|
Set-StrictMode -Version Latest
|
|
|
|
$sourceCount = 16
|
|
$faultIndexes = @(5, 6, 7, 8)
|
|
$sampleSeconds = 10
|
|
$mediaMTXVersion = 'v1.19.3'
|
|
$mediaMTXSHA256 = '5d82148d1032a6a190d9909a2997d9989457aaadf49af87dd02cd4512d31bebe'
|
|
$mediaMTXExecutableSHA256 = '1cda85249312cb9463f9f94c5a712b9f160c9af3fd9490f0d4723911d7880e05'
|
|
$migrationNames = @(
|
|
'002_bell.sql',
|
|
'003_sense.sql',
|
|
'004_privileges.sql',
|
|
'005_area_policy.sql',
|
|
'006_device_operation_outbox.sql',
|
|
'007_privileges_area_audit.sql',
|
|
'008_control_api.sql',
|
|
'009_privileges_control_api.sql',
|
|
'010_reconcile_safety.sql',
|
|
'011_privileges_reconcile_safety.sql'
|
|
)
|
|
|
|
function ConvertTo-Base64Url([byte[]]$Bytes) {
|
|
return [Convert]::ToBase64String($Bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')
|
|
}
|
|
|
|
function New-RandomBytes([int]$Count) {
|
|
$bytes = [byte[]]::new($Count)
|
|
[Security.Cryptography.RandomNumberGenerator]::Fill($bytes)
|
|
return $bytes
|
|
}
|
|
|
|
function New-SecretToken {
|
|
return ConvertTo-Base64Url (New-RandomBytes 32)
|
|
}
|
|
|
|
function Get-SHA256Hex([string]$Value) {
|
|
$bytes = [Text.Encoding]::UTF8.GetBytes($Value)
|
|
$digest = [Security.Cryptography.SHA256]::HashData($bytes)
|
|
return ([Convert]::ToHexString($digest)).ToLowerInvariant()
|
|
}
|
|
|
|
function Get-FreeTcpPort([Collections.Generic.HashSet[int]]$Reserved) {
|
|
while ($true) {
|
|
$listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, 0)
|
|
$listener.Start()
|
|
$port = ([Net.IPEndPoint]$listener.LocalEndpoint).Port
|
|
$listener.Stop()
|
|
if ($Reserved.Add($port)) {
|
|
return $port
|
|
}
|
|
}
|
|
}
|
|
|
|
function Wait-Port([int]$Port, [int]$TimeoutSeconds = 30) {
|
|
$watch = [Diagnostics.Stopwatch]::StartNew()
|
|
while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
|
|
$client = [Net.Sockets.TcpClient]::new()
|
|
try {
|
|
$task = $client.ConnectAsync('127.0.0.1', $Port)
|
|
if ($task.Wait(500) -and $client.Connected) {
|
|
return
|
|
}
|
|
}
|
|
catch {
|
|
}
|
|
finally {
|
|
$client.Dispose()
|
|
}
|
|
Start-Sleep -Milliseconds 250
|
|
}
|
|
throw "local process did not listen before timeout"
|
|
}
|
|
|
|
function Start-ManagedProcess(
|
|
[string]$Name,
|
|
[string]$FilePath,
|
|
[string[]]$Arguments,
|
|
[hashtable]$Environment = @{}
|
|
) {
|
|
$start = [Diagnostics.ProcessStartInfo]::new()
|
|
$start.FileName = $FilePath
|
|
$start.WorkingDirectory = $script:session
|
|
$start.UseShellExecute = $false
|
|
$start.CreateNoWindow = $true
|
|
$start.RedirectStandardOutput = $true
|
|
$start.RedirectStandardError = $true
|
|
foreach ($argument in $Arguments) {
|
|
$start.ArgumentList.Add($argument)
|
|
}
|
|
foreach ($entry in $Environment.GetEnumerator()) {
|
|
$start.Environment[$entry.Key] = [string]$entry.Value
|
|
}
|
|
$process = [Diagnostics.Process]::new()
|
|
$process.StartInfo = $start
|
|
if (-not $process.Start()) {
|
|
throw "failed to start required process: $Name"
|
|
}
|
|
return [pscustomobject]@{
|
|
Name = $Name
|
|
Process = $process
|
|
Stdout = $process.StandardOutput.ReadToEndAsync()
|
|
Stderr = $process.StandardError.ReadToEndAsync()
|
|
}
|
|
}
|
|
|
|
function Stop-ManagedProcess($Managed) {
|
|
if ($null -eq $Managed -or $null -eq $Managed.Process) {
|
|
return
|
|
}
|
|
if (-not $Managed.Process.HasExited) {
|
|
$Managed.Process.Kill()
|
|
$Managed.Process.WaitForExit(10000) | Out-Null
|
|
}
|
|
}
|
|
|
|
function Stop-AllManagedProcesses($Items) {
|
|
$processes = @($Items | Where-Object { $null -ne $_ -and $null -ne $_.Process })
|
|
foreach ($item in $processes) {
|
|
try {
|
|
if (-not $item.Process.HasExited) {
|
|
$item.Process.Kill()
|
|
}
|
|
}
|
|
catch {
|
|
if (-not $item.Process.HasExited) { throw }
|
|
}
|
|
}
|
|
$cleanupWatch = [Diagnostics.Stopwatch]::StartNew()
|
|
while ($cleanupWatch.Elapsed.TotalSeconds -lt 10 -and
|
|
@($processes | Where-Object { -not $_.Process.HasExited }).Count -gt 0) {
|
|
Start-Sleep -Milliseconds 100
|
|
}
|
|
$remaining = @($processes | Where-Object { -not $_.Process.HasExited })
|
|
if ($remaining.Count -gt 0) {
|
|
Write-Warning "managed processes did not exit within the shared cleanup deadline: $($remaining.Count)"
|
|
}
|
|
foreach ($item in $processes) {
|
|
if ($item.Process.HasExited) {
|
|
$item.Stdout.Wait(2000) | Out-Null
|
|
$item.Stderr.Wait(2000) | Out-Null
|
|
$item.Process.Dispose()
|
|
}
|
|
}
|
|
}
|
|
|
|
function Remove-SessionDirectory([string]$Path) {
|
|
Get-ChildItem -LiteralPath $Path -Recurse -Force -ErrorAction SilentlyContinue |
|
|
ForEach-Object { $_.Attributes = 'Normal' }
|
|
(Get-Item -LiteralPath $Path -Force).Attributes = 'Directory'
|
|
$deleteWatch = [Diagnostics.Stopwatch]::StartNew()
|
|
while (Test-Path -LiteralPath $Path) {
|
|
try {
|
|
Remove-Item -LiteralPath $Path -Recurse -Force -ErrorAction Stop
|
|
}
|
|
catch {
|
|
if ($deleteWatch.Elapsed.TotalSeconds -ge 10) { throw }
|
|
Start-Sleep -Milliseconds 200
|
|
}
|
|
}
|
|
}
|
|
|
|
function Assert-Alive($Managed) {
|
|
if ($null -eq $Managed -or $Managed.Process.HasExited) {
|
|
$code = if ($null -eq $Managed) { 'not-started' } else { $Managed.Process.ExitCode }
|
|
throw "required process exited: $($Managed.Name), code=$code"
|
|
}
|
|
}
|
|
|
|
function Invoke-Checked {
|
|
param(
|
|
[Parameter(Mandatory)] [string]$FilePath,
|
|
[Parameter(ValueFromRemainingArguments)] [string[]]$Arguments
|
|
)
|
|
& $FilePath @Arguments 2>&1 | Out-Null
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "required command failed with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Invoke-ControlRequest(
|
|
[string]$Method,
|
|
[string]$Path,
|
|
[int[]]$ExpectedStatus,
|
|
$Body = $null,
|
|
[hashtable]$ExtraHeaders = @{}
|
|
) {
|
|
$headers = @{
|
|
Authorization = "Bearer $script:controlToken"
|
|
Accept = 'application/json'
|
|
}
|
|
foreach ($entry in $ExtraHeaders.GetEnumerator()) {
|
|
$headers[$entry.Key] = $entry.Value
|
|
}
|
|
$parameters = @{
|
|
Method = $Method
|
|
Uri = "http://127.0.0.1:$script:senseHTTPPort$Path"
|
|
Headers = $headers
|
|
TimeoutSec = 15
|
|
NoProxy = $true
|
|
SkipHttpErrorCheck = $true
|
|
}
|
|
if ($null -ne $Body) {
|
|
$parameters.ContentType = 'application/json'
|
|
$parameters.Body = ($Body | ConvertTo-Json -Depth 10 -Compress)
|
|
}
|
|
$response = Invoke-WebRequest @parameters
|
|
$statusCode = [int]$response.StatusCode
|
|
$statusMatches = $false
|
|
foreach ($expected in $ExpectedStatus) {
|
|
if ([int]$expected -eq $statusCode) {
|
|
$statusMatches = $true
|
|
break
|
|
}
|
|
}
|
|
if (-not $statusMatches) {
|
|
$code = 'unknown'
|
|
try {
|
|
$code = ((Get-ResponseText $response) | ConvertFrom-Json).code
|
|
}
|
|
catch {
|
|
}
|
|
throw "Control API returned unexpected status $statusCode, expected=$($ExpectedStatus -join ','), code=$code"
|
|
}
|
|
return $response
|
|
}
|
|
|
|
function Get-ResponseText($Response) {
|
|
if ($Response.Content -is [byte[]]) {
|
|
return [Text.Encoding]::UTF8.GetString($Response.Content)
|
|
}
|
|
return [string]$Response.Content
|
|
}
|
|
|
|
function Convert-ResponseJSON($Response) {
|
|
return ((Get-ResponseText $Response) | ConvertFrom-Json)
|
|
}
|
|
|
|
function Get-ResponseETag($Response) {
|
|
$value = [string]$Response.Headers.ETag
|
|
if ([string]::IsNullOrWhiteSpace($value)) {
|
|
throw 'Control API response did not contain ETag'
|
|
}
|
|
return $value
|
|
}
|
|
|
|
function Get-DevicePage {
|
|
$response = Invoke-ControlRequest 'GET' '/api/v1/sites/site-t014/devices?limit=100' @(200)
|
|
return Convert-ResponseJSON $response
|
|
}
|
|
|
|
function Get-DeviceEntries([string[]]$DeviceIDs) {
|
|
$entries = [Collections.Generic.List[object]]::new()
|
|
foreach ($id in $DeviceIDs) {
|
|
$response = Invoke-ControlRequest 'GET' "/api/v1/sites/site-t014/devices/$id" @(200)
|
|
$entries.Add([pscustomobject]@{
|
|
DeviceID = $id
|
|
ETag = Get-ResponseETag $response
|
|
})
|
|
}
|
|
return @($entries)
|
|
}
|
|
|
|
function Invoke-BatchDesiredState(
|
|
[string]$DesiredState,
|
|
[object[]]$Entries,
|
|
[string]$KeySuffix
|
|
) {
|
|
$items = @($Entries | ForEach-Object {
|
|
[ordered]@{
|
|
device_id = $_.DeviceID
|
|
etag = $_.ETag
|
|
desired_state = $DesiredState
|
|
}
|
|
})
|
|
$watch = [Diagnostics.Stopwatch]::StartNew()
|
|
$response = Invoke-ControlRequest 'POST' '/api/v1/sites/site-t014/devices:batchDesiredState' @(202) ([ordered]@{
|
|
items = $items
|
|
reason = "T-014 laboratory $DesiredState baseline"
|
|
}) @{ 'Idempotency-Key' = "t014-batch-$KeySuffix-$script:runNonce" }
|
|
$operation = Convert-ResponseJSON $response
|
|
$results = @($operation.results)
|
|
if ($operation.status -ne 'succeeded' -or $results.Count -ne $sourceCount -or
|
|
@($results | Where-Object { $_.status -ne 'succeeded' }).Count -ne 0) {
|
|
$statusCounts = @($results | Group-Object status | Sort-Object Name |
|
|
ForEach-Object { "$($_.Name):$($_.Count)" }) -join ','
|
|
$errorCounts = @($results | Where-Object { $_.error_code } | Group-Object error_code | Sort-Object Name |
|
|
ForEach-Object { "$($_.Name):$($_.Count)" }) -join ','
|
|
throw "batch desired-state operation was not fully successful: operation=$($operation.status), items=$($results.Count), statuses=$statusCounts, errors=$errorCounts"
|
|
}
|
|
return [ordered]@{
|
|
status = [string]$operation.status
|
|
item_count = $results.Count
|
|
request_seconds = [Math]::Round($watch.Elapsed.TotalSeconds, 3)
|
|
}
|
|
}
|
|
|
|
function Get-MediaInventory([int]$APIPort) {
|
|
$config = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$APIPort/v3/config/paths/list?page=0&itemsPerPage=100" -TimeoutSec 5 -NoProxy
|
|
$runtime = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$APIPort/v3/paths/list?page=0&itemsPerPage=100" -TimeoutSec 5 -NoProxy
|
|
$items = @($runtime.items)
|
|
[UInt64]$inboundBytes = 0
|
|
[UInt64]$frameErrors = 0
|
|
foreach ($item in $items) {
|
|
if ($null -ne $item.inboundBytes) {
|
|
$inboundBytes += [UInt64]$item.inboundBytes
|
|
}
|
|
if ($null -ne $item.inboundFramesInError) {
|
|
$frameErrors += [UInt64]$item.inboundFramesInError
|
|
}
|
|
}
|
|
return [pscustomobject]@{
|
|
Configured = [int]$config.itemCount
|
|
Runtime = [int]$runtime.itemCount
|
|
Online = @($items | Where-Object { $_.online -eq $true -and $_.available -eq $true }).Count
|
|
InboundBytes = $inboundBytes
|
|
FrameErrors = $frameErrors
|
|
}
|
|
}
|
|
|
|
function Get-SenseMetric([string]$Name, [string]$Labels) {
|
|
$response = Invoke-WebRequest -Method Get -Uri "http://127.0.0.1:$script:senseHTTPPort/metrics" -TimeoutSec 5 -NoProxy
|
|
$prefix = [Regex]::Escape($Name + $Labels)
|
|
$match = [Regex]::Match((Get-ResponseText $response), "(?m)^$prefix\s+([-+0-9.eE]+)$")
|
|
if (-not $match.Success) {
|
|
throw "required Sense metric was not found: $Name"
|
|
}
|
|
return [double]::Parse($match.Groups[1].Value, [Globalization.CultureInfo]::InvariantCulture)
|
|
}
|
|
|
|
function Wait-SenseHealth($Managed, [int]$TimeoutSeconds = 45) {
|
|
$watch = [Diagnostics.Stopwatch]::StartNew()
|
|
while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
|
|
Assert-Alive $Managed
|
|
try {
|
|
$ready = Invoke-RestMethod -Method Get -Uri "http://127.0.0.1:$script:senseHTTPPort/readyz" -TimeoutSec 2 -NoProxy
|
|
if ($ready.status -eq 'ready') {
|
|
return
|
|
}
|
|
}
|
|
catch {
|
|
}
|
|
Start-Sleep -Milliseconds 500
|
|
}
|
|
throw 'Sense readiness endpoint did not become ready'
|
|
}
|
|
|
|
function Wait-SourcePublishers([int]$TimeoutSeconds = 90) {
|
|
$watch = [Diagnostics.Stopwatch]::StartNew()
|
|
while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
|
|
foreach ($publisher in $script:publishers.Values) {
|
|
Assert-Alive $publisher
|
|
}
|
|
try {
|
|
$inventory = Get-MediaInventory $script:sourceAPIPort
|
|
if ($inventory.Runtime -eq $sourceCount -and $inventory.Online -eq $sourceCount) {
|
|
return [Math]::Round($watch.Elapsed.TotalSeconds, 1)
|
|
}
|
|
}
|
|
catch {
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
throw 'independent source publishers did not all become online'
|
|
}
|
|
|
|
function Wait-Convergence([ValidateSet('enabled', 'disabled')] [string]$DesiredState, [int]$TimeoutSeconds = 300) {
|
|
$watch = [Diagnostics.Stopwatch]::StartNew()
|
|
$last = 'no observation'
|
|
while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
|
|
try {
|
|
$page = Get-DevicePage
|
|
$devices = @($page.items)
|
|
$inventory = Get-MediaInventory $script:productionAPIPort
|
|
$totalMetric = Get-SenseMetric 'sense_reconcile_devices' '{state="total"}'
|
|
$unconvergedMetric = Get-SenseMetric 'sense_reconcile_devices' '{state="unconverged"}'
|
|
if ($DesiredState -eq 'enabled') {
|
|
$stateOK = $devices.Count -eq $sourceCount -and
|
|
@($devices | Where-Object {
|
|
$_.desired_state -ne 'enabled' -or $_.actual_state -ne 'online' -or -not $_.converged
|
|
}).Count -eq 0
|
|
$mediaOK = $inventory.Configured -eq $sourceCount -and
|
|
$inventory.Runtime -eq $sourceCount -and $inventory.Online -eq $sourceCount
|
|
$metricsOK = $totalMetric -eq $sourceCount -and $unconvergedMetric -eq 0
|
|
}
|
|
else {
|
|
$stateOK = $devices.Count -eq $sourceCount -and
|
|
@($devices | Where-Object {
|
|
$_.desired_state -ne 'disabled' -or $_.actual_state -ne 'offline' -or -not $_.converged
|
|
}).Count -eq 0
|
|
$mediaOK = $inventory.Configured -eq 0 -and $inventory.Runtime -eq 0
|
|
$metricsOK = $totalMetric -eq 0 -and $unconvergedMetric -eq 0
|
|
}
|
|
$last = "devices=$($devices.Count), configured=$($inventory.Configured), runtime=$($inventory.Runtime), online=$($inventory.Online), unconverged=$unconvergedMetric"
|
|
if ($stateOK -and $mediaOK -and $metricsOK) {
|
|
return [Math]::Round($watch.Elapsed.TotalSeconds, 1)
|
|
}
|
|
}
|
|
catch {
|
|
$last = $_.Exception.Message
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
throw "devices did not reach $DesiredState convergence: $last"
|
|
}
|
|
|
|
function Wait-PartialFault([int]$TimeoutSeconds = 120) {
|
|
$watch = [Diagnostics.Stopwatch]::StartNew()
|
|
while ($watch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
|
|
try {
|
|
$page = Get-DevicePage
|
|
$devices = @($page.items)
|
|
$online = @($devices | Where-Object { $_.actual_state -eq 'online' }).Count
|
|
$notOnline = $devices.Count - $online
|
|
if ($devices.Count -eq $sourceCount -and $online -eq ($sourceCount - $faultIndexes.Count) -and
|
|
$notOnline -eq $faultIndexes.Count) {
|
|
return [ordered]@{
|
|
detection_seconds = [Math]::Round($watch.Elapsed.TotalSeconds, 1)
|
|
affected = $notOnline
|
|
unaffected_online = $online
|
|
}
|
|
}
|
|
}
|
|
catch {
|
|
}
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
throw 'four-source fault was not isolated to exactly four devices'
|
|
}
|
|
|
|
function Start-Publisher([int]$Index) {
|
|
$name = 'synthetic-{0:D2}' -f $Index
|
|
return Start-ManagedProcess "publisher-$Index" $script:ffmpeg @(
|
|
'-hide_banner', '-loglevel', 'warning', '-re', '-stream_loop', '-1',
|
|
'-i', $script:fixturePath, '-map', '0:v:0', '-c', 'copy', '-an',
|
|
'-f', 'rtsp', '-rtsp_transport', 'tcp',
|
|
"rtsp://127.0.0.1:$script:sourceRTSPPort/$name"
|
|
)
|
|
}
|
|
|
|
function Get-ProcessResourceSample($Managed, [double]$PreviousCPU, [double]$ElapsedSeconds) {
|
|
Assert-Alive $Managed
|
|
$Managed.Process.Refresh()
|
|
$totalCPU = $Managed.Process.TotalProcessorTime.TotalSeconds
|
|
$normalizedCPU = 0.0
|
|
if ($ElapsedSeconds -gt 0) {
|
|
$normalizedCPU = (($totalCPU - $PreviousCPU) / $ElapsedSeconds / [Environment]::ProcessorCount) * 100
|
|
}
|
|
return [pscustomobject]@{
|
|
TotalCPU = $totalCPU
|
|
CPUPercent = [Math]::Max(0, $normalizedCPU)
|
|
WorkingSetMiB = $Managed.Process.WorkingSet64 / 1MB
|
|
PrivateMiB = $Managed.Process.PrivateMemorySize64 / 1MB
|
|
Handles = $Managed.Process.HandleCount
|
|
}
|
|
}
|
|
|
|
function Get-Average([Collections.Generic.List[double]]$Values) {
|
|
if ($Values.Count -eq 0) { return 0.0 }
|
|
return ($Values | Measure-Object -Average).Average
|
|
}
|
|
|
|
function Get-Maximum([Collections.Generic.List[double]]$Values) {
|
|
if ($Values.Count -eq 0) { return 0.0 }
|
|
return ($Values | Measure-Object -Maximum).Maximum
|
|
}
|
|
|
|
function Get-PostgresScalar([string]$Query) {
|
|
$value = & $script:psql -X -A -t -v ON_ERROR_STOP=1 -d $script:adminDatabaseDSN -c $Query
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw 'PostgreSQL observation query failed'
|
|
}
|
|
return (($value | Out-String).Trim())
|
|
}
|
|
|
|
function Write-Result([string]$JSON) {
|
|
if (-not [string]::IsNullOrWhiteSpace($OutputPath)) {
|
|
$resolvedOutput = [IO.Path]::GetFullPath($OutputPath)
|
|
$repoPrefix = $script:repoRoot.TrimEnd('\') + '\'
|
|
if ($resolvedOutput.StartsWith($repoPrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
|
throw 'capacity result output must stay outside the repository'
|
|
}
|
|
$parent = Split-Path -Parent $resolvedOutput
|
|
if (-not (Test-Path -LiteralPath $parent -PathType Container)) {
|
|
throw 'capacity result output directory does not exist'
|
|
}
|
|
[IO.File]::WriteAllText($resolvedOutput, $JSON + [Environment]::NewLine, [Text.UTF8Encoding]::new($false))
|
|
}
|
|
Write-Output $JSON
|
|
}
|
|
|
|
$script:repoRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '..\..')).Path
|
|
$pgRootResolved = (Resolve-Path -LiteralPath $PgRoot).Path
|
|
$pgBin = Join-Path $pgRootResolved 'bin'
|
|
$script:initdb = Join-Path $pgBin 'initdb.exe'
|
|
$script:pgCtl = Join-Path $pgBin 'pg_ctl.exe'
|
|
$script:psql = Join-Path $pgBin 'psql.exe'
|
|
$script:createdb = Join-Path $pgBin 'createdb.exe'
|
|
$postgres = Join-Path $pgBin 'postgres.exe'
|
|
foreach ($required in @($script:initdb, $script:pgCtl, $script:psql, $script:createdb, $postgres)) {
|
|
if (-not (Test-Path -LiteralPath $required -PathType Leaf)) {
|
|
throw 'PostgreSQL executable is missing under the selected PgRoot'
|
|
}
|
|
}
|
|
$postgresVersion = (& $postgres --version 2>&1 | Out-String).Trim()
|
|
if ($LASTEXITCODE -ne 0 -or $postgresVersion -notmatch 'PostgreSQL\) 17\.10$') {
|
|
throw 'T-014 requires the frozen PostgreSQL 17.10 binaries'
|
|
}
|
|
$script:ffmpeg = (Get-Command ffmpeg -ErrorAction Stop).Source
|
|
$ffmpegVersion = (& $script:ffmpeg -version 2>&1 | Select-Object -First 1)
|
|
if ($ffmpegVersion -notmatch '^ffmpeg version 8\.1\.2') {
|
|
throw 'T-014 requires the frozen FFmpeg 8.1.2 build'
|
|
}
|
|
$senseDirectory = Join-Path $script:repoRoot 'Sense'
|
|
$goVersion = (& go '-C' $senseDirectory 'version' 2>&1 | Out-String).Trim()
|
|
if ($LASTEXITCODE -ne 0 -or $goVersion -notmatch '^go version go1\.26\.5 ') {
|
|
throw 'T-014 requires the frozen Sense Go 1.26.5 toolchain'
|
|
}
|
|
|
|
$systemTemp = [IO.Path]::GetFullPath([IO.Path]::GetTempPath()).TrimEnd('\')
|
|
$runtimeResolved = [IO.Path]::GetFullPath($RuntimeRoot).TrimEnd('\')
|
|
$runtimePrefix = $systemTemp + '\'
|
|
if (-not $runtimeResolved.StartsWith($runtimePrefix, [StringComparison]::OrdinalIgnoreCase) -or
|
|
[IO.Path]::GetFileName($runtimeResolved) -notlike 'yovision-t014*') {
|
|
throw 'RuntimeRoot must be a yovision-t014 directory under the system temporary directory'
|
|
}
|
|
|
|
$mediaDirectory = Join-Path $runtimeResolved 'mediamtx-v1.19.3'
|
|
$mediaMTX = Join-Path $mediaDirectory 'mediamtx.exe'
|
|
if ($PreflightOnly) {
|
|
$memoryGiB = 0.0
|
|
try { $memoryGiB = (Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB } catch {}
|
|
Write-Result (([ordered]@{
|
|
success = $true
|
|
preflight_only = $true
|
|
source_count = $sourceCount
|
|
observation_minutes = $ObservationMinutes
|
|
formal_eligible = $ObservationMinutes -ge 30
|
|
mediamtx_cached = Test-Path -LiteralPath $mediaMTX -PathType Leaf
|
|
postgres_version = $postgresVersion
|
|
ffmpeg_version = [string]$ffmpegVersion
|
|
go_version = $goVersion
|
|
logical_processors = [Environment]::ProcessorCount
|
|
total_memory_gib = [Math]::Round($memoryGiB, 2)
|
|
} | ConvertTo-Json -Depth 5))
|
|
exit 0
|
|
}
|
|
|
|
New-Item -ItemType Directory -Path $runtimeResolved -Force | Out-Null
|
|
if (-not (Test-Path -LiteralPath $mediaMTX -PathType Leaf)) {
|
|
$zip = Join-Path $runtimeResolved 'mediamtx_v1.19.3_windows_amd64.zip'
|
|
Invoke-WebRequest 'https://github.com/bluenviron/mediamtx/releases/download/v1.19.3/mediamtx_v1.19.3_windows_amd64.zip' -OutFile $zip
|
|
$actualHash = (Get-FileHash -LiteralPath $zip -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
if ($actualHash -ne $mediaMTXSHA256) {
|
|
throw 'MediaMTX checksum mismatch'
|
|
}
|
|
New-Item -ItemType Directory -Path $mediaDirectory -Force | Out-Null
|
|
Expand-Archive -LiteralPath $zip -DestinationPath $mediaDirectory -Force
|
|
}
|
|
if ((Get-FileHash -LiteralPath $mediaMTX -Algorithm SHA256).Hash.ToLowerInvariant() -ne $mediaMTXExecutableSHA256) {
|
|
throw 'MediaMTX executable checksum mismatch'
|
|
}
|
|
|
|
$script:runNonce = [Guid]::NewGuid().ToString('N')
|
|
$script:session = Join-Path $runtimeResolved ("session-$script:runNonce")
|
|
New-Item -ItemType Directory -Path $script:session | Out-Null
|
|
$reservedPorts = [Collections.Generic.HashSet[int]]::new()
|
|
$script:sourceRTSPPort = Get-FreeTcpPort $reservedPorts
|
|
$script:sourceAPIPort = Get-FreeTcpPort $reservedPorts
|
|
$productionRTSPPort = Get-FreeTcpPort $reservedPorts
|
|
$script:productionAPIPort = Get-FreeTcpPort $reservedPorts
|
|
$productionMetricsPort = Get-FreeTcpPort $reservedPorts
|
|
$script:senseHTTPPort = Get-FreeTcpPort $reservedPorts
|
|
$postgresPort = Get-FreeTcpPort $reservedPorts
|
|
|
|
$sourceConfig = Join-Path $script:session 'mediamtx-source.yml'
|
|
$productionConfig = Join-Path $script:session 'mediamtx-production.yml'
|
|
$sourceLines = [Collections.Generic.List[string]]::new()
|
|
@(
|
|
'logLevel: warn',
|
|
"rtspAddress: 127.0.0.1:$script:sourceRTSPPort",
|
|
'rtspTransports: [tcp]',
|
|
'api: true',
|
|
"apiAddress: 127.0.0.1:$script:sourceAPIPort",
|
|
'metrics: false', 'rtmp: false', 'hls: false', 'webrtc: false', 'srt: false', 'moq: false',
|
|
'paths:'
|
|
) | ForEach-Object { $sourceLines.Add($_) }
|
|
foreach ($index in 1..$sourceCount) {
|
|
$sourceLines.Add((' synthetic-{0:D2}:' -f $index))
|
|
$sourceLines.Add(' source: publisher')
|
|
}
|
|
[IO.File]::WriteAllLines($sourceConfig, $sourceLines, [Text.UTF8Encoding]::new($false))
|
|
[IO.File]::WriteAllLines($productionConfig, @(
|
|
'logLevel: warn',
|
|
"rtspAddress: 127.0.0.1:$productionRTSPPort",
|
|
'rtspTransports: [tcp]',
|
|
'api: true',
|
|
"apiAddress: 127.0.0.1:$script:productionAPIPort",
|
|
'metrics: true',
|
|
"metricsAddress: 127.0.0.1:$productionMetricsPort",
|
|
'rtmp: false', 'hls: false', 'webrtc: false', 'srt: false', 'moq: false',
|
|
'paths: {}'
|
|
), [Text.UTF8Encoding]::new($false))
|
|
|
|
$script:fixturePath = Join-Path $script:session 'fixture.mp4'
|
|
$senseBinary = Join-Path $script:session 'sense-api.exe'
|
|
$authPath = Join-Path $script:session 'control-auth.json'
|
|
$cursorPath = Join-Path $script:session 'cursor.key'
|
|
$pgData = Join-Path $script:session 'pgdata'
|
|
$pgLog = Join-Path $script:session 'postgres.log'
|
|
$databaseName = 'yovision_t014'
|
|
$adminRootDSN = "postgres://postgres@127.0.0.1:$postgresPort/postgres?sslmode=disable"
|
|
$script:adminDatabaseDSN = "postgres://postgres@127.0.0.1:$postgresPort/${databaseName}?sslmode=disable"
|
|
$senseDSN = "postgres://yovision_t014_sense@127.0.0.1:$postgresPort/${databaseName}?sslmode=disable"
|
|
$existing5432 = @(
|
|
Get-NetTCPConnection -State Listen -LocalPort 5432 -ErrorAction SilentlyContinue |
|
|
Select-Object -ExpandProperty OwningProcess -Unique | Sort-Object
|
|
)
|
|
|
|
$managed = [Collections.Generic.List[object]]::new()
|
|
$script:publishers = @{}
|
|
$postgresStarted = $false
|
|
$postgresStopped = $false
|
|
$success = $false
|
|
$stage = 'preparing fixtures'
|
|
$failure = $null
|
|
$resultJSON = $null
|
|
try {
|
|
Invoke-Checked $script:ffmpeg '-hide_banner' '-loglevel' 'error' '-f' 'lavfi' '-i' 'testsrc2=size=640x360:rate=10' '-t' '30' '-c:v' 'libx264' '-preset' 'ultrafast' '-tune' 'zerolatency' '-pix_fmt' 'yuv420p' '-g' '10' '-an' '-movflags' '+faststart' '-y' $script:fixturePath
|
|
Invoke-Checked 'go' '-C' $senseDirectory 'build' '-o' $senseBinary './cmd/sense-api'
|
|
|
|
$stage = 'starting isolated PostgreSQL'
|
|
New-Item -ItemType Directory -Path $pgData | Out-Null
|
|
Invoke-Checked $script:initdb '-D' $pgData '-U' 'postgres' '-A' 'trust' '--encoding=UTF8' '--no-locale' '--no-sync'
|
|
$serverOptions = "-h 127.0.0.1 -p $postgresPort -c listen_addresses=127.0.0.1"
|
|
# Do not pipe pg_ctl start output: postgres can inherit the pipeline handle and
|
|
# keep PowerShell waiting until the server exits.
|
|
& $script:pgCtl '-D' $pgData '-l' $pgLog '-o' $serverOptions '-w' 'start'
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "required command failed with exit code $LASTEXITCODE"
|
|
}
|
|
$postgresStarted = $true
|
|
Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-f' (Join-Path $script:repoRoot 'deploy\postgres\001_roles.sql')
|
|
Invoke-Checked $script:createdb '-h' '127.0.0.1' '-p' ([string]$postgresPort) '-U' 'postgres' $databaseName
|
|
foreach ($pass in 1..2) {
|
|
foreach ($name in $migrationNames) {
|
|
Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $script:adminDatabaseDSN '-f' (Join-Path $script:repoRoot "deploy\postgres\$name")
|
|
}
|
|
}
|
|
Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $adminRootDSN '-c' 'CREATE ROLE yovision_t014_sense LOGIN IN ROLE sense_app'
|
|
Invoke-Checked $script:psql '-X' '-v' 'ON_ERROR_STOP=1' '-d' $script:adminDatabaseDSN '-c' "INSERT INTO bell.sites(tenant_id,id,name,max_video_channels) VALUES ('tenant-t014','site-t014','T-014 Lab',16); INSERT INTO bell.areas(tenant_id,site_id,id,name,capture_policy) VALUES ('tenant-t014','site-t014','area-t014','T-014 Area','video_allowed');"
|
|
|
|
$stage = 'starting independent publishers'
|
|
$sourceMedia = Start-ManagedProcess 'mediamtx-source' $mediaMTX @($sourceConfig)
|
|
$managed.Add($sourceMedia)
|
|
Wait-Port $script:sourceAPIPort
|
|
foreach ($index in 1..$sourceCount) {
|
|
$script:publishers[$index] = Start-Publisher $index
|
|
$managed.Add($script:publishers[$index])
|
|
}
|
|
$publisherReadySeconds = Wait-SourcePublishers
|
|
$productionMedia = Start-ManagedProcess 'mediamtx-production' $mediaMTX @($productionConfig)
|
|
$managed.Add($productionMedia)
|
|
Wait-Port $script:productionAPIPort
|
|
|
|
$stage = 'starting Sense Control API'
|
|
$script:controlToken = New-SecretToken
|
|
$tokenDigest = Get-SHA256Hex $script:controlToken
|
|
$cursorKey = ConvertTo-Base64Url (New-RandomBytes 32)
|
|
$localCredential = New-SecretToken
|
|
[IO.File]::WriteAllText($authPath, (([ordered]@{
|
|
version = 1
|
|
principals = @([ordered]@{
|
|
token_sha256 = $tokenDigest
|
|
subject_id = 't014-runner'
|
|
actor_type = 'service'
|
|
tenant_id = 'tenant-t014'
|
|
site_ids = @('site-t014')
|
|
permissions = @('sense.devices.read', 'sense.devices.write')
|
|
})
|
|
} | ConvertTo-Json -Depth 6) + [Environment]::NewLine), [Text.UTF8Encoding]::new($false))
|
|
[IO.File]::WriteAllText($cursorPath, $cursorKey + [Environment]::NewLine, [Text.UTF8Encoding]::new($false))
|
|
$senseEnvironment = @{
|
|
SENSE_HTTP_ADDR = "127.0.0.1:$script:senseHTTPPort"
|
|
SENSE_DB_DRIVER = 'postgres'
|
|
SENSE_DB_DSN = $senseDSN
|
|
SENSE_MEDIAMTX_URL = "http://127.0.0.1:$script:productionAPIPort"
|
|
SENSE_RECONCILE_INTERVAL = '1s'
|
|
SENSE_RECONCILE_LEASE_DURATION = '30s'
|
|
SENSE_RECONCILE_OPERATION_TIMEOUT = '20s'
|
|
SENSE_PROBE_INTERVAL = '1s'
|
|
SENSE_INSTANCE_ID = 't014-lab'
|
|
SENSE_METRICS_ENABLED = 'true'
|
|
SENSE_ORPHAN_SCAN_ENABLED = 'false'
|
|
SENSE_ONVIF_MODE = 'disabled'
|
|
SENSE_CONTROL_API_ENABLED = 'true'
|
|
SENSE_CONTROL_AUTH_FILE = $authPath
|
|
SENSE_CONTROL_CURSOR_KEY_FILE = $cursorPath
|
|
SENSE_CREDENTIAL_SYNTHETIC_ONVIF_USERNAME = 't014-local'
|
|
SENSE_CREDENTIAL_SYNTHETIC_ONVIF_PASSWORD = $localCredential
|
|
SENSE_CREDENTIAL_SYNTHETIC_RTSP_USERNAME = 't014-local'
|
|
SENSE_CREDENTIAL_SYNTHETIC_RTSP_PASSWORD = $localCredential
|
|
}
|
|
$sense = Start-ManagedProcess 'sense-api' $senseBinary @() $senseEnvironment
|
|
$managed.Add($sense)
|
|
Wait-SenseHealth $sense
|
|
|
|
$stage = 'creating 16 disabled devices'
|
|
$deviceIDs = [Collections.Generic.List[string]]::new()
|
|
foreach ($index in 1..$sourceCount) {
|
|
$name = 'synthetic-{0:D2}' -f $index
|
|
$response = Invoke-ControlRequest 'POST' '/api/v1/sites/site-t014/devices' @(201) ([ordered]@{
|
|
serial_number = "t014-$name"
|
|
name = "T-014 source $index"
|
|
modality = 'video'
|
|
capabilities = @('video_capture')
|
|
area_id = 'area-t014'
|
|
endpoint_ref = "rtsp://127.0.0.1:$script:sourceRTSPPort/$name"
|
|
credential_ref = 'env://synthetic'
|
|
desired_state = 'disabled'
|
|
}) @{ 'Idempotency-Key' = "t014-create-$index-$script:runNonce" }
|
|
$created = Convert-ResponseJSON $response
|
|
$deviceIDs.Add([string]$created.id)
|
|
}
|
|
$stage = 'batch enabling 16 devices'
|
|
$firstEnableEntries = Get-DeviceEntries @($deviceIDs)
|
|
$firstEnableBatch = Invoke-BatchDesiredState 'enabled' $firstEnableEntries 'enable-1'
|
|
$firstEnableSeconds = Wait-Convergence 'enabled'
|
|
$firstEnabledInventory = Get-MediaInventory $script:productionAPIPort
|
|
|
|
$stage = 'rejecting seventeenth enabled device'
|
|
$overflowResponse = Invoke-ControlRequest 'POST' '/api/v1/sites/site-t014/devices' @(409) ([ordered]@{
|
|
serial_number = 't014-overflow-17'
|
|
name = 'T-014 overflow source'
|
|
modality = 'video'
|
|
capabilities = @('video_capture')
|
|
area_id = 'area-t014'
|
|
endpoint_ref = "rtsp://127.0.0.1:$script:sourceRTSPPort/synthetic-01"
|
|
credential_ref = 'env://synthetic'
|
|
desired_state = 'enabled'
|
|
}) @{ 'Idempotency-Key' = "t014-overflow-17-$script:runNonce" }
|
|
$overflowProblem = Convert-ResponseJSON $overflowResponse
|
|
$overflowCodeProperty = $overflowProblem.PSObject.Properties['code']
|
|
if ($null -eq $overflowCodeProperty) {
|
|
$propertyNames = @($overflowProblem.PSObject.Properties.Name) -join ','
|
|
throw "quota response did not contain a stable code field; properties=$propertyNames"
|
|
}
|
|
$overflowCode = [string]$overflowCodeProperty.Value
|
|
$pageAfterOverflow = Get-DevicePage
|
|
if ($overflowCode -ne 'quota_exceeded' -or @($pageAfterOverflow.items).Count -ne $sourceCount) {
|
|
throw 'the seventeenth enabled device did not fail closed at the site quota'
|
|
}
|
|
|
|
$stage = 'batch disabling 16 devices'
|
|
$disableEntries = Get-DeviceEntries @($deviceIDs)
|
|
$disableBatch = Invoke-BatchDesiredState 'disabled' $disableEntries 'disable'
|
|
$disableSeconds = Wait-Convergence 'disabled'
|
|
$disabledInventory = Get-MediaInventory $script:productionAPIPort
|
|
|
|
$stage = 'batch re-enabling 16 devices'
|
|
$secondEnableEntries = Get-DeviceEntries @($deviceIDs)
|
|
$secondEnableBatch = Invoke-BatchDesiredState 'enabled' $secondEnableEntries 'enable-2'
|
|
$secondEnableSeconds = Wait-Convergence 'enabled'
|
|
$secondEnabledInventory = Get-MediaInventory $script:productionAPIPort
|
|
|
|
$stage = 'injecting four-source fault'
|
|
foreach ($index in $faultIndexes) {
|
|
Stop-ManagedProcess $script:publishers[$index]
|
|
}
|
|
$fault = Wait-PartialFault
|
|
foreach ($index in $faultIndexes) {
|
|
$script:publishers[$index] = Start-Publisher $index
|
|
$managed.Add($script:publishers[$index])
|
|
}
|
|
$faultRecoverySeconds = Wait-Convergence 'enabled'
|
|
$fault.recovery_seconds = $faultRecoverySeconds
|
|
$fault.final_unconverged = 0
|
|
|
|
$stage = 'observing stable 16-stream capacity'
|
|
$observation = [Diagnostics.Stopwatch]::StartNew()
|
|
$targetSeconds = $ObservationMinutes * 60
|
|
$senseCPU = [Collections.Generic.List[double]]::new()
|
|
$senseWorking = [Collections.Generic.List[double]]::new()
|
|
$sensePrivate = [Collections.Generic.List[double]]::new()
|
|
$senseHandles = [Collections.Generic.List[double]]::new()
|
|
$mediaCPU = [Collections.Generic.List[double]]::new()
|
|
$mediaWorking = [Collections.Generic.List[double]]::new()
|
|
$mediaPrivate = [Collections.Generic.List[double]]::new()
|
|
$mediaHandles = [Collections.Generic.List[double]]::new()
|
|
$databaseConnections = [Collections.Generic.List[double]]::new()
|
|
$previousSenseCPU = $sense.Process.TotalProcessorTime.TotalSeconds
|
|
$previousMediaCPU = $productionMedia.Process.TotalProcessorTime.TotalSeconds
|
|
$observationSenseCPUStart = $previousSenseCPU
|
|
$observationMediaCPUStart = $previousMediaCPU
|
|
$previousSampleAt = 0.0
|
|
$initialInventory = Get-MediaInventory $script:productionAPIPort
|
|
$samples = 0
|
|
$requiredSamples = [int][Math]::Ceiling($targetSeconds / $sampleSeconds)
|
|
$maximumUnconverged = 0
|
|
while ($samples -lt $requiredSamples) {
|
|
$nextSampleAt = ($samples + 1) * $sampleSeconds
|
|
$delayMilliseconds = [int][Math]::Ceiling(($nextSampleAt - $observation.Elapsed.TotalSeconds) * 1000)
|
|
if ($delayMilliseconds -gt 0) {
|
|
Start-Sleep -Milliseconds $delayMilliseconds
|
|
}
|
|
Assert-Alive $sourceMedia
|
|
Assert-Alive $productionMedia
|
|
Assert-Alive $sense
|
|
foreach ($publisher in $script:publishers.Values) { Assert-Alive $publisher }
|
|
$elapsed = $observation.Elapsed.TotalSeconds
|
|
$delta = $elapsed - $previousSampleAt
|
|
$senseSample = Get-ProcessResourceSample $sense $previousSenseCPU $delta
|
|
$mediaSample = Get-ProcessResourceSample $productionMedia $previousMediaCPU $delta
|
|
$previousSenseCPU = $senseSample.TotalCPU
|
|
$previousMediaCPU = $mediaSample.TotalCPU
|
|
$previousSampleAt = $elapsed
|
|
$senseCPU.Add($senseSample.CPUPercent)
|
|
$senseWorking.Add($senseSample.WorkingSetMiB)
|
|
$sensePrivate.Add($senseSample.PrivateMiB)
|
|
$senseHandles.Add($senseSample.Handles)
|
|
$mediaCPU.Add($mediaSample.CPUPercent)
|
|
$mediaWorking.Add($mediaSample.WorkingSetMiB)
|
|
$mediaPrivate.Add($mediaSample.PrivateMiB)
|
|
$mediaHandles.Add($mediaSample.Handles)
|
|
$databaseConnections.Add([double](Get-PostgresScalar "SELECT count(*) FROM pg_stat_activity WHERE datname = '$databaseName'"))
|
|
$page = Get-DevicePage
|
|
$inventory = Get-MediaInventory $script:productionAPIPort
|
|
$unconverged = [int](Get-SenseMetric 'sense_reconcile_devices' '{state="unconverged"}')
|
|
$maximumUnconverged = [Math]::Max($maximumUnconverged, $unconverged)
|
|
if (@($page.items).Count -ne $sourceCount -or
|
|
@($page.items | Where-Object { $_.desired_state -ne 'enabled' -or $_.actual_state -ne 'online' -or -not $_.converged }).Count -ne 0 -or
|
|
$inventory.Configured -ne $sourceCount -or $inventory.Runtime -ne $sourceCount -or
|
|
$inventory.Online -ne $sourceCount -or $unconverged -ne 0) {
|
|
throw 'stability observation detected a non-converged stream'
|
|
}
|
|
$samples++
|
|
}
|
|
$observationSeconds = [Math]::Round($observation.Elapsed.TotalSeconds, 1)
|
|
$finalInventory = Get-MediaInventory $script:productionAPIPort
|
|
$sense.Process.Refresh()
|
|
$productionMedia.Process.Refresh()
|
|
$senseCPUTotal = $sense.Process.TotalProcessorTime.TotalSeconds - $observationSenseCPUStart
|
|
$mediaCPUTotal = $productionMedia.Process.TotalProcessorTime.TotalSeconds - $observationMediaCPUStart
|
|
$inboundDelta = [UInt64]($finalInventory.InboundBytes - $initialInventory.InboundBytes)
|
|
$frameErrorDelta = [UInt64]($finalInventory.FrameErrors - $initialInventory.FrameErrors)
|
|
$inboundMbps = if ($observationSeconds -gt 0) { ($inboundDelta * 8.0) / $observationSeconds / 1000000.0 } else { 0.0 }
|
|
$databaseSizeBytes = [Int64](Get-PostgresScalar "SELECT pg_database_size('$databaseName')")
|
|
$finalUnconverged = [int](Get-SenseMetric 'sense_reconcile_devices' '{state="unconverged"}')
|
|
if ($observationSeconds -lt $targetSeconds -or $finalUnconverged -ne 0 -or
|
|
$maximumUnconverged -ne 0 -or $samples -lt $requiredSamples) {
|
|
throw 'formal observation did not satisfy sampling and convergence requirements'
|
|
}
|
|
|
|
$memoryGiB = 0.0
|
|
try { $memoryGiB = (Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB } catch {}
|
|
$gitCommit = (& git -C $script:repoRoot rev-parse HEAD | Out-String).Trim()
|
|
$success = $true
|
|
$resultJSON = [ordered]@{
|
|
success = $true
|
|
formal_eligible = $ObservationMinutes -ge 30
|
|
scope = 'laboratory_software_baseline'
|
|
source_count = $sourceCount
|
|
independent_publishers = $sourceCount
|
|
fixture = [ordered]@{ width = 640; height = 360; fps = 10; codec = 'H.264'; audio = $false; publish_mode = 'preencoded_copy' }
|
|
versions = [ordered]@{
|
|
repository_commit = $gitCommit
|
|
sense_sha256 = (Get-FileHash -LiteralPath $senseBinary -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
mediamtx = $mediaMTXVersion
|
|
mediamtx_package_sha256 = $mediaMTXSHA256
|
|
mediamtx_executable_sha256 = $mediaMTXExecutableSHA256
|
|
postgres = $postgresVersion
|
|
ffmpeg = [string]$ffmpegVersion
|
|
go = $goVersion
|
|
}
|
|
host = [ordered]@{ logical_processors = [Environment]::ProcessorCount; total_memory_gib = [Math]::Round($memoryGiB, 2) }
|
|
quota = [ordered]@{ configured = 16; accepted_devices = 16; seventeenth_rejected = $true; error_code = 'quota_exceeded' }
|
|
publishers_ready_seconds = $publisherReadySeconds
|
|
batch = [ordered]@{
|
|
enable_first = $firstEnableBatch
|
|
disable = $disableBatch
|
|
enable_second = $secondEnableBatch
|
|
}
|
|
convergence_seconds = [ordered]@{
|
|
enable_first = $firstEnableSeconds
|
|
disable = $disableSeconds
|
|
enable_second = $secondEnableSeconds
|
|
}
|
|
configured_paths = [ordered]@{
|
|
after_enable_first = $firstEnabledInventory.Configured
|
|
after_disable = $disabledInventory.Configured
|
|
after_enable_second = $secondEnabledInventory.Configured
|
|
}
|
|
fault_recovery = $fault
|
|
observation = [ordered]@{
|
|
requested_minutes = $ObservationMinutes
|
|
seconds = $observationSeconds
|
|
sample_interval_seconds = $sampleSeconds
|
|
samples = $samples
|
|
maximum_unconverged = $maximumUnconverged
|
|
final_unconverged = $finalUnconverged
|
|
final_online_paths = $finalInventory.Online
|
|
aggregate_inbound_mbps = [Math]::Round($inboundMbps, 3)
|
|
inbound_frame_errors = $frameErrorDelta
|
|
}
|
|
resources = [ordered]@{
|
|
sense = [ordered]@{
|
|
cpu_average_percent = [Math]::Round((Get-Average $senseCPU), 3)
|
|
cpu_peak_percent = [Math]::Round((Get-Maximum $senseCPU), 3)
|
|
cpu_total_seconds = [Math]::Round($senseCPUTotal, 3)
|
|
working_set_peak_mib = [Math]::Round((Get-Maximum $senseWorking), 2)
|
|
private_peak_mib = [Math]::Round((Get-Maximum $sensePrivate), 2)
|
|
handles_peak = [int](Get-Maximum $senseHandles)
|
|
}
|
|
mediamtx = [ordered]@{
|
|
cpu_average_percent = [Math]::Round((Get-Average $mediaCPU), 3)
|
|
cpu_peak_percent = [Math]::Round((Get-Maximum $mediaCPU), 3)
|
|
cpu_total_seconds = [Math]::Round($mediaCPUTotal, 3)
|
|
working_set_peak_mib = [Math]::Round((Get-Maximum $mediaWorking), 2)
|
|
private_peak_mib = [Math]::Round((Get-Maximum $mediaPrivate), 2)
|
|
handles_peak = [int](Get-Maximum $mediaHandles)
|
|
}
|
|
postgres_peak_connections = [int](Get-Maximum $databaseConnections)
|
|
postgres_database_mib = [Math]::Round($databaseSizeBytes / 1MB, 2)
|
|
}
|
|
limitations = @('synthetic_sources', 'no_customer_network', 'no_recording', 'no_downstream_readers', 'no_ai_or_gpu', 'not_a_production_sla')
|
|
} | ConvertTo-Json -Depth 12
|
|
}
|
|
catch {
|
|
$failure = "T-014 stage '$stage' failed: $($_.Exception.Message)"
|
|
}
|
|
finally {
|
|
Stop-AllManagedProcesses @($managed)
|
|
if ($postgresStarted) {
|
|
& $script:pgCtl '-D' $pgData '-m' 'fast' '-w' 'stop' | Out-Null
|
|
$postgresStopped = $LASTEXITCODE -eq 0
|
|
}
|
|
if (-not $postgresStarted -or $postgresStopped) {
|
|
$resolvedSession = [IO.Path]::GetFullPath($script:session)
|
|
$expectedPrefix = $runtimeResolved.TrimEnd('\') + '\'
|
|
if (-not $resolvedSession.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase) -or
|
|
[IO.Path]::GetFileName($resolvedSession) -notlike 'session-*') {
|
|
throw 'refusing to clean an unexpected T-014 session path'
|
|
}
|
|
if (Test-Path -LiteralPath $resolvedSession) {
|
|
Remove-SessionDirectory $resolvedSession
|
|
}
|
|
}
|
|
else {
|
|
Write-Warning 'Temporary PostgreSQL did not stop; the session directory was retained for manual recovery'
|
|
}
|
|
$after5432 = @(
|
|
Get-NetTCPConnection -State Listen -LocalPort 5432 -ErrorAction SilentlyContinue |
|
|
Select-Object -ExpandProperty OwningProcess -Unique | Sort-Object
|
|
)
|
|
if (($existing5432 -join ',') -ne ($after5432 -join ',')) {
|
|
throw 'the existing PostgreSQL listener on port 5432 changed during T-014'
|
|
}
|
|
}
|
|
if (-not $success) {
|
|
throw $failure
|
|
}
|
|
Write-Result $resultJSON
|